From 991031ba3563dc4026dabbece70effc2b06d63cf Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Wed, 13 Nov 2024 09:49:30 +0100 Subject: [PATCH 01/18] started refactor --- dendrite/__init__.py | 4 +- .../async_api/_api/dto/get_interaction_dto.py | 10 -- .../_common/_exceptions/__init__.py | 0 .../_common/_exceptions/_constants.py | 0 .../_common/_exceptions/dendrite_exception.py | 2 +- dendrite/{ => browser}/async_api/__init__.py | 0 .../{ => browser}/async_api/_api/__init__.py | 0 .../async_api/_api/_http_client.py | 3 +- .../async_api/_api/browser_api_client.py | 34 ++--- .../async_api/_api/dto/__init__.py | 0 .../async_api}/_api/dto/ask_page_dto.py | 4 +- .../async_api/_api/dto/authenticate_dto.py | 0 .../async_api/_api/dto/extract_dto.py | 4 +- .../async_api}/_api/dto/get_elements_dto.py | 5 +- .../async_api/_api/dto/get_interaction_dto.py | 10 ++ .../async_api/_api/dto/get_session_dto.py | 0 .../async_api/_api/dto/google_search_dto.py | 4 +- .../_api/dto/make_interaction_dto.py | 4 +- .../async_api/_api/dto/try_run_script_dto.py | 2 +- .../_api/dto/upload_auth_session_dto.py | 2 +- .../async_api/_api/response/__init__.py | 0 .../_api/response/ask_page_response.py | 0 .../_api/response/cache_extract_response.py | 0 .../_api/response/extract_response.py | 4 +- .../_api/response/get_element_response.py | 2 +- .../_api/response/google_search_response.py | 0 .../_api/response/interaction_response.py | 2 +- .../_api/response/selector_cache_response.py | 0 .../_api/response/session_response.py | 0 .../async_api/_common/__init__.py | 0 .../async_api/_common/constants.py | 0 .../async_api/_common/event_sync.py | 0 .../{ => browser}/async_api/_common/status.py | 0 .../{ => browser}/async_api/_core/__init__.py | 0 .../async_api/_core/_impl_browser.py | 4 +- .../async_api/_core/_impl_mapping.py | 12 +- .../async_api/_core/_js/__init__.py | 0 .../async_api/_core/_js/eventListenerPatch.js | 0 .../_core/_js/generateDendriteIDs.js | 65 +++++----- .../_core/_js/generateDendriteIDsIframe.js | 67 +++++----- .../async_api/_core/_managers/__init__.py | 0 .../_core/_managers/navigation_tracker.py | 2 +- .../async_api/_core/_managers/page_manager.py | 6 +- .../_core/_managers/screenshot_manager.py | 2 +- .../async_api/_core/_type_spec.py | 0 .../{ => browser}/async_api/_core/_utils.py | 14 +- .../async_api/_core/dendrite_browser.py | 51 ++++---- .../async_api/_core/dendrite_element.py | 19 +-- .../async_api/_core/dendrite_page.py | 41 +++--- .../async_api/_core/mixin/ask.py | 8 +- .../async_api/_core/mixin/click.py | 8 +- .../async_api/_core/mixin/extract.py | 12 +- .../async_api/_core/mixin/fill_fields.py | 8 +- .../async_api/_core/mixin/get_element.py | 16 +-- .../async_api/_core/mixin/keyboard.py | 4 +- .../async_api/_core/mixin/markdown.py | 4 +- .../async_api/_core/mixin/screenshot.py | 2 +- .../async_api/_core/mixin/wait_for.py | 8 +- .../async_api/_core/models/__init__.py | 0 .../async_api/_core/models/api_config.py | 2 +- .../async_api/_core/models/authentication.py | 0 .../_core/models/download_interface.py | 0 .../_core/models/page_diff_information.py | 2 +- .../_core/models/page_information.py | 0 .../async_api/_core/models/response.py | 2 +- .../async_api/_core/protocol/page_protocol.py | 6 +- .../{ => browser}/async_api/_dom/__init__.py | 0 .../async_api/_dom/util/mild_strip.py | 0 .../async_api/_ext_impl/__init__.py | 0 .../_ext_impl/browserbase/__init__.py | 0 .../_ext_impl/browserbase/_client.py | 2 +- .../_ext_impl/browserbase/_download.py | 4 +- .../async_api/_ext_impl/browserbase/_impl.py | 14 +- .../_ext_impl/browserless/__init__.py | 0 .../async_api/_ext_impl/browserless/_impl.py | 14 +- dendrite/browser/remote/__init__.py | 8 ++ .../remote/browserbase_config.py | 0 .../remote/browserless_config.py | 2 +- dendrite/{ => browser}/remote/provider.py | 4 +- dendrite/{ => browser}/sync_api/__init__.py | 0 .../{ => browser}/sync_api/_api/__init__.py | 0 .../sync_api/_api/_http_client.py | 2 +- .../sync_api/_api/browser_api_client.py | 34 ++--- .../sync_api/_api/dto/__init__.py | 0 .../sync_api}/_api/dto/ask_page_dto.py | 4 +- .../sync_api/_api/dto/authenticate_dto.py | 0 .../sync_api/_api/dto/extract_dto.py | 4 +- .../sync_api}/_api/dto/get_elements_dto.py | 5 +- .../sync_api/_api/dto/get_interaction_dto.py | 9 ++ .../sync_api/_api/dto/get_session_dto.py | 0 .../sync_api/_api/dto/google_search_dto.py | 4 +- .../sync_api/_api/dto/make_interaction_dto.py | 4 +- .../sync_api/_api/dto/try_run_script_dto.py | 2 +- .../_api/dto/upload_auth_session_dto.py | 2 +- .../sync_api/_api/response/__init__.py | 0 .../_api/response/ask_page_response.py | 0 .../_api/response/cache_extract_response.py | 0 .../_api/response/extract_response.py | 4 +- .../_api/response/get_element_response.py | 2 +- .../_api/response/google_search_response.py | 0 .../_api/response/interaction_response.py | 2 +- .../_api/response/selector_cache_response.py | 0 .../_api/response/session_response.py | 0 .../sync_api/_common/__init__.py | 0 .../sync_api/_common/constants.py | 0 .../sync_api/_common/event_sync.py | 0 .../{ => browser}/sync_api/_common/status.py | 0 .../{ => browser}/sync_api/_core/__init__.py | 0 .../sync_api/_core/_impl_browser.py | 4 +- .../sync_api/_core/_impl_mapping.py | 12 +- .../sync_api/_core/_js/__init__.py | 0 .../sync_api/_core/_js/eventListenerPatch.js | 0 .../_core/_js/generateDendriteIDs.js | 0 .../_core/_js/generateDendriteIDsIframe.js | 0 .../sync_api/_core/_managers/__init__.py | 0 .../_core/_managers/navigation_tracker.py | 2 +- .../sync_api/_core/_managers/page_manager.py | 6 +- .../_core/_managers/screenshot_manager.py | 2 +- .../sync_api/_core/_type_spec.py | 0 .../{ => browser}/sync_api/_core/_utils.py | 14 +- .../sync_api/_core/dendrite_browser.py | 46 +++---- .../sync_api/_core/dendrite_element.py | 16 +-- .../sync_api/_core/dendrite_page.py | 34 ++--- .../{ => browser}/sync_api/_core/mixin/ask.py | 8 +- .../sync_api/_core/mixin/click.py | 8 +- .../sync_api/_core/mixin/extract.py | 12 +- .../sync_api/_core/mixin/fill_fields.py | 8 +- .../sync_api/_core/mixin/get_element.py | 17 +-- .../sync_api/_core/mixin/keyboard.py | 4 +- .../sync_api/_core/mixin/markdown.py | 4 +- .../sync_api/_core/mixin/screenshot.py | 2 +- .../sync_api/_core/mixin/wait_for.py | 8 +- .../sync_api/_core/models/__init__.py | 0 .../sync_api/_core/models/api_config.py | 2 +- .../sync_api/_core/models/authentication.py | 0 .../_core/models/download_interface.py | 0 .../_core/models/page_diff_information.py | 2 +- .../sync_api/_core/models/page_information.py | 0 .../sync_api/_core/models/response.py | 2 +- .../sync_api/_core/protocol/page_protocol.py | 6 +- .../{ => browser}/sync_api/_dom/__init__.py | 0 .../sync_api/_dom/util/mild_strip.py | 0 .../sync_api/_ext_impl/__init__.py | 0 .../_ext_impl/browserbase/__init__.py | 0 .../sync_api/_ext_impl/browserbase/_client.py | 2 +- .../_ext_impl/browserbase/_download.py | 4 +- .../sync_api/_ext_impl/browserbase/_impl.py | 14 +- .../_ext_impl/browserless/__init__.py | 0 .../sync_api/_ext_impl/browserless/_impl.py | 14 +- dendrite/exceptions/__init__.py | 2 +- dendrite/logic/hosted/_api/__init__.py | 0 dendrite/logic/hosted/_api/_http_client.py | 65 ++++++++++ .../logic/hosted/_api/browser_api_client.py | 120 ++++++++++++++++++ dendrite/logic/hosted/_api/dto/__init__.py | 0 .../logic/hosted/_api/dto/ask_page_dto.py | 11 ++ .../logic/hosted/_api/dto/authenticate_dto.py | 6 + dendrite/logic/hosted/_api/dto/extract_dto.py | 25 ++++ .../logic/hosted/_api/dto/get_elements_dto.py | 19 +++ .../hosted/_api/dto/get_interaction_dto.py | 10 ++ .../logic/hosted/_api/dto/get_session_dto.py | 7 + .../hosted/_api/dto/google_search_dto.py | 12 ++ .../hosted/_api/dto/make_interaction_dto.py | 19 +++ .../hosted/_api/dto/try_run_script_dto.py | 14 ++ .../_api/dto/upload_auth_session_dto.py | 11 ++ .../logic/hosted/_api/response/__init__.py | 0 .../hosted/_api/response/ask_page_response.py | 11 ++ .../_api/response/cache_extract_response.py | 5 + .../hosted/_api/response/extract_response.py | 15 +++ .../_api/response/get_element_response.py | 12 ++ .../_api/response/google_search_response.py | 12 ++ .../_api/response/interaction_response.py | 7 + .../_api/response/selector_cache_response.py | 5 + .../hosted/_api/response/session_response.py | 7 + dendrite/logic/hosted/async_api_impl.py | 0 dendrite/logic/interfaces/async_api.py | 48 +++++++ dendrite/remote/__init__.py | 8 -- .../sync_api/_api/dto/get_interaction_dto.py | 9 -- scripts/generate_sync.py | 4 +- tests/tests_async/conftest.py | 4 +- tests/tests_async/test_browserbase.py | 2 +- tests/tests_async/test_download.py | 2 +- tests/tests_sync/conftest.py | 2 +- tests/tests_sync/test_context.py | 2 +- tests/tests_sync/test_download.py | 2 +- 184 files changed, 884 insertions(+), 434 deletions(-) delete mode 100644 dendrite/async_api/_api/dto/get_interaction_dto.py rename dendrite/{ => browser}/_common/_exceptions/__init__.py (100%) rename dendrite/{ => browser}/_common/_exceptions/_constants.py (100%) rename dendrite/{ => browser}/_common/_exceptions/dendrite_exception.py (98%) rename dendrite/{ => browser}/async_api/__init__.py (100%) rename dendrite/{ => browser}/async_api/_api/__init__.py (100%) rename dendrite/{ => browser}/async_api/_api/_http_client.py (96%) rename dendrite/{ => browser}/async_api/_api/browser_api_client.py (71%) rename dendrite/{ => browser}/async_api/_api/dto/__init__.py (100%) rename dendrite/{sync_api => browser/async_api}/_api/dto/ask_page_dto.py (56%) rename dendrite/{ => browser}/async_api/_api/dto/authenticate_dto.py (100%) rename dendrite/{ => browser}/async_api/_api/dto/extract_dto.py (79%) rename dendrite/{sync_api => browser/async_api}/_api/dto/get_elements_dto.py (70%) create mode 100644 dendrite/browser/async_api/_api/dto/get_interaction_dto.py rename dendrite/{ => browser}/async_api/_api/dto/get_session_dto.py (100%) rename dendrite/{ => browser}/async_api/_api/dto/google_search_dto.py (61%) rename dendrite/{ => browser}/async_api/_api/dto/make_interaction_dto.py (72%) rename dendrite/{ => browser}/async_api/_api/dto/try_run_script_dto.py (81%) rename dendrite/{ => browser}/async_api/_api/dto/upload_auth_session_dto.py (71%) rename dendrite/{ => browser}/async_api/_api/response/__init__.py (100%) rename dendrite/{ => browser}/async_api/_api/response/ask_page_response.py (100%) rename dendrite/{ => browser}/async_api/_api/response/cache_extract_response.py (100%) rename dendrite/{sync_api => browser/async_api}/_api/response/extract_response.py (80%) rename dendrite/{ => browser}/async_api/_api/response/get_element_response.py (80%) rename dendrite/{ => browser}/async_api/_api/response/google_search_response.py (100%) rename dendrite/{sync_api => browser/async_api}/_api/response/interaction_response.py (63%) rename dendrite/{ => browser}/async_api/_api/response/selector_cache_response.py (100%) rename dendrite/{ => browser}/async_api/_api/response/session_response.py (100%) rename dendrite/{ => browser}/async_api/_common/__init__.py (100%) rename dendrite/{ => browser}/async_api/_common/constants.py (100%) rename dendrite/{ => browser}/async_api/_common/event_sync.py (100%) rename dendrite/{ => browser}/async_api/_common/status.py (100%) rename dendrite/{ => browser}/async_api/_core/__init__.py (100%) rename dendrite/{ => browser}/async_api/_core/_impl_browser.py (93%) rename dendrite/{ => browser}/async_api/_core/_impl_mapping.py (64%) rename dendrite/{ => browser}/async_api/_core/_js/__init__.py (100%) rename dendrite/{ => browser}/async_api/_core/_js/eventListenerPatch.js (100%) rename dendrite/{sync_api => browser/async_api}/_core/_js/generateDendriteIDs.js (62%) rename dendrite/{sync_api => browser/async_api}/_core/_js/generateDendriteIDsIframe.js (63%) rename dendrite/{ => browser}/async_api/_core/_managers/__init__.py (100%) rename dendrite/{ => browser}/async_api/_core/_managers/navigation_tracker.py (97%) rename dendrite/{ => browser}/async_api/_core/_managers/page_manager.py (93%) rename dendrite/{ => browser}/async_api/_core/_managers/screenshot_manager.py (96%) rename dendrite/{ => browser}/async_api/_core/_type_spec.py (100%) rename dendrite/{ => browser}/async_api/_core/_utils.py (87%) rename dendrite/{ => browser}/async_api/_core/dendrite_browser.py (89%) rename dendrite/{ => browser}/async_api/_core/dendrite_element.py (92%) rename dendrite/{ => browser}/async_api/_core/dendrite_page.py (89%) rename dendrite/{ => browser}/async_api/_core/mixin/ask.py (96%) rename dendrite/{ => browser}/async_api/_core/mixin/click.py (86%) rename dendrite/{ => browser}/async_api/_core/mixin/extract.py (94%) rename dendrite/{ => browser}/async_api/_core/mixin/fill_fields.py (90%) rename dendrite/{ => browser}/async_api/_core/mixin/get_element.py (95%) rename dendrite/{ => browser}/async_api/_core/mixin/keyboard.py (91%) rename dendrite/{ => browser}/async_api/_core/mixin/markdown.py (89%) rename dendrite/{ => browser}/async_api/_core/mixin/screenshot.py (87%) rename dendrite/{ => browser}/async_api/_core/mixin/wait_for.py (87%) rename dendrite/{ => browser}/async_api/_core/models/__init__.py (100%) rename dendrite/{ => browser}/async_api/_core/models/api_config.py (93%) rename dendrite/{ => browser}/async_api/_core/models/authentication.py (100%) rename dendrite/{ => browser}/async_api/_core/models/download_interface.py (100%) rename dendrite/{ => browser}/async_api/_core/models/page_diff_information.py (61%) rename dendrite/{ => browser}/async_api/_core/models/page_information.py (100%) rename dendrite/{ => browser}/async_api/_core/models/response.py (96%) rename dendrite/{ => browser}/async_api/_core/protocol/page_protocol.py (63%) rename dendrite/{ => browser}/async_api/_dom/__init__.py (100%) rename dendrite/{ => browser}/async_api/_dom/util/mild_strip.py (100%) rename dendrite/{ => browser}/async_api/_ext_impl/__init__.py (100%) rename dendrite/{ => browser}/async_api/_ext_impl/browserbase/__init__.py (100%) rename dendrite/{ => browser}/async_api/_ext_impl/browserbase/_client.py (97%) rename dendrite/{ => browser}/async_api/_ext_impl/browserbase/_download.py (92%) rename dendrite/{ => browser}/async_api/_ext_impl/browserbase/_impl.py (80%) rename dendrite/{ => browser}/async_api/_ext_impl/browserless/__init__.py (100%) rename dendrite/{ => browser}/async_api/_ext_impl/browserless/_impl.py (74%) create mode 100644 dendrite/browser/remote/__init__.py rename dendrite/{ => browser}/remote/browserbase_config.py (100%) rename dendrite/{ => browser}/remote/browserless_config.py (88%) rename dendrite/{ => browser}/remote/provider.py (92%) rename dendrite/{ => browser}/sync_api/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_api/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_api/_http_client.py (96%) rename dendrite/{ => browser}/sync_api/_api/browser_api_client.py (69%) rename dendrite/{ => browser}/sync_api/_api/dto/__init__.py (100%) rename dendrite/{async_api => browser/sync_api}/_api/dto/ask_page_dto.py (57%) rename dendrite/{ => browser}/sync_api/_api/dto/authenticate_dto.py (100%) rename dendrite/{ => browser}/sync_api/_api/dto/extract_dto.py (79%) rename dendrite/{async_api => browser/sync_api}/_api/dto/get_elements_dto.py (70%) create mode 100644 dendrite/browser/sync_api/_api/dto/get_interaction_dto.py rename dendrite/{ => browser}/sync_api/_api/dto/get_session_dto.py (100%) rename dendrite/{ => browser}/sync_api/_api/dto/google_search_dto.py (62%) rename dendrite/{ => browser}/sync_api/_api/dto/make_interaction_dto.py (69%) rename dendrite/{ => browser}/sync_api/_api/dto/try_run_script_dto.py (77%) rename dendrite/{ => browser}/sync_api/_api/dto/upload_auth_session_dto.py (58%) rename dendrite/{ => browser}/sync_api/_api/response/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_api/response/ask_page_response.py (100%) rename dendrite/{ => browser}/sync_api/_api/response/cache_extract_response.py (100%) rename dendrite/{async_api => browser/sync_api}/_api/response/extract_response.py (81%) rename dendrite/{ => browser}/sync_api/_api/response/get_element_response.py (81%) rename dendrite/{ => browser}/sync_api/_api/response/google_search_response.py (100%) rename dendrite/{async_api => browser/sync_api}/_api/response/interaction_response.py (64%) rename dendrite/{ => browser}/sync_api/_api/response/selector_cache_response.py (100%) rename dendrite/{ => browser}/sync_api/_api/response/session_response.py (100%) rename dendrite/{ => browser}/sync_api/_common/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_common/constants.py (100%) rename dendrite/{ => browser}/sync_api/_common/event_sync.py (100%) rename dendrite/{ => browser}/sync_api/_common/status.py (100%) rename dendrite/{ => browser}/sync_api/_core/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_core/_impl_browser.py (93%) rename dendrite/{ => browser}/sync_api/_core/_impl_mapping.py (63%) rename dendrite/{ => browser}/sync_api/_core/_js/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_core/_js/eventListenerPatch.js (100%) rename dendrite/{async_api => browser/sync_api}/_core/_js/generateDendriteIDs.js (100%) rename dendrite/{async_api => browser/sync_api}/_core/_js/generateDendriteIDsIframe.js (100%) rename dendrite/{ => browser}/sync_api/_core/_managers/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_core/_managers/navigation_tracker.py (97%) rename dendrite/{ => browser}/sync_api/_core/_managers/page_manager.py (93%) rename dendrite/{ => browser}/sync_api/_core/_managers/screenshot_manager.py (96%) rename dendrite/{ => browser}/sync_api/_core/_type_spec.py (100%) rename dendrite/{ => browser}/sync_api/_core/_utils.py (86%) rename dendrite/{ => browser}/sync_api/_core/dendrite_browser.py (90%) rename dendrite/{ => browser}/sync_api/_core/dendrite_element.py (92%) rename dendrite/{ => browser}/sync_api/_core/dendrite_page.py (90%) rename dendrite/{ => browser}/sync_api/_core/mixin/ask.py (96%) rename dendrite/{ => browser}/sync_api/_core/mixin/click.py (85%) rename dendrite/{ => browser}/sync_api/_core/mixin/extract.py (93%) rename dendrite/{ => browser}/sync_api/_core/mixin/fill_fields.py (89%) rename dendrite/{ => browser}/sync_api/_core/mixin/get_element.py (94%) rename dendrite/{ => browser}/sync_api/_core/mixin/keyboard.py (91%) rename dendrite/{ => browser}/sync_api/_core/mixin/markdown.py (87%) rename dendrite/{ => browser}/sync_api/_core/mixin/screenshot.py (87%) rename dendrite/{ => browser}/sync_api/_core/mixin/wait_for.py (86%) rename dendrite/{ => browser}/sync_api/_core/models/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_core/models/api_config.py (93%) rename dendrite/{ => browser}/sync_api/_core/models/authentication.py (100%) rename dendrite/{ => browser}/sync_api/_core/models/download_interface.py (100%) rename dendrite/{ => browser}/sync_api/_core/models/page_diff_information.py (61%) rename dendrite/{ => browser}/sync_api/_core/models/page_information.py (100%) rename dendrite/{ => browser}/sync_api/_core/models/response.py (96%) rename dendrite/{ => browser}/sync_api/_core/protocol/page_protocol.py (63%) rename dendrite/{ => browser}/sync_api/_dom/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_dom/util/mild_strip.py (100%) rename dendrite/{ => browser}/sync_api/_ext_impl/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_ext_impl/browserbase/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_ext_impl/browserbase/_client.py (96%) rename dendrite/{ => browser}/sync_api/_ext_impl/browserbase/_download.py (91%) rename dendrite/{ => browser}/sync_api/_ext_impl/browserbase/_impl.py (78%) rename dendrite/{ => browser}/sync_api/_ext_impl/browserless/__init__.py (100%) rename dendrite/{ => browser}/sync_api/_ext_impl/browserless/_impl.py (72%) create mode 100644 dendrite/logic/hosted/_api/__init__.py create mode 100644 dendrite/logic/hosted/_api/_http_client.py create mode 100644 dendrite/logic/hosted/_api/browser_api_client.py create mode 100644 dendrite/logic/hosted/_api/dto/__init__.py create mode 100644 dendrite/logic/hosted/_api/dto/ask_page_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/authenticate_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/extract_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/get_elements_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/get_interaction_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/get_session_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/google_search_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/make_interaction_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/try_run_script_dto.py create mode 100644 dendrite/logic/hosted/_api/dto/upload_auth_session_dto.py create mode 100644 dendrite/logic/hosted/_api/response/__init__.py create mode 100644 dendrite/logic/hosted/_api/response/ask_page_response.py create mode 100644 dendrite/logic/hosted/_api/response/cache_extract_response.py create mode 100644 dendrite/logic/hosted/_api/response/extract_response.py create mode 100644 dendrite/logic/hosted/_api/response/get_element_response.py create mode 100644 dendrite/logic/hosted/_api/response/google_search_response.py create mode 100644 dendrite/logic/hosted/_api/response/interaction_response.py create mode 100644 dendrite/logic/hosted/_api/response/selector_cache_response.py create mode 100644 dendrite/logic/hosted/_api/response/session_response.py create mode 100644 dendrite/logic/hosted/async_api_impl.py create mode 100644 dendrite/logic/interfaces/async_api.py delete mode 100644 dendrite/remote/__init__.py delete mode 100644 dendrite/sync_api/_api/dto/get_interaction_dto.py diff --git a/dendrite/__init__.py b/dendrite/__init__.py index 931c728..1e5852c 100644 --- a/dendrite/__init__.py +++ b/dendrite/__init__.py @@ -1,13 +1,13 @@ import sys from loguru import logger -from dendrite.async_api import ( +from dendrite.browser.async_api import ( AsyncDendrite, AsyncElement, AsyncPage, AsyncElementsResponse, ) -from dendrite.sync_api import ( +from dendrite.browser.sync_api import ( Dendrite, Element, Page, diff --git a/dendrite/async_api/_api/dto/get_interaction_dto.py b/dendrite/async_api/_api/dto/get_interaction_dto.py deleted file mode 100644 index 1d93432..0000000 --- a/dendrite/async_api/_api/dto/get_interaction_dto.py +++ /dev/null @@ -1,10 +0,0 @@ -from pydantic import BaseModel - -from dendrite.async_api._core.models.api_config import APIConfig -from dendrite.async_api._core.models.page_information import PageInformation - - -class GetInteractionDTO(BaseModel): - page_information: PageInformation - api_config: APIConfig - prompt: str diff --git a/dendrite/_common/_exceptions/__init__.py b/dendrite/browser/_common/_exceptions/__init__.py similarity index 100% rename from dendrite/_common/_exceptions/__init__.py rename to dendrite/browser/_common/_exceptions/__init__.py diff --git a/dendrite/_common/_exceptions/_constants.py b/dendrite/browser/_common/_exceptions/_constants.py similarity index 100% rename from dendrite/_common/_exceptions/_constants.py rename to dendrite/browser/_common/_exceptions/_constants.py diff --git a/dendrite/_common/_exceptions/dendrite_exception.py b/dendrite/browser/_common/_exceptions/dendrite_exception.py similarity index 98% rename from dendrite/_common/_exceptions/dendrite_exception.py rename to dendrite/browser/_common/_exceptions/dendrite_exception.py index 4d62481..1eaa160 100644 --- a/dendrite/_common/_exceptions/dendrite_exception.py +++ b/dendrite/browser/_common/_exceptions/dendrite_exception.py @@ -5,7 +5,7 @@ from loguru import logger -from dendrite._common._exceptions._constants import INVALID_AUTH_SESSION_MSG +from dendrite.browser._common._exceptions._constants import INVALID_AUTH_SESSION_MSG class BaseDendriteException(Exception): diff --git a/dendrite/async_api/__init__.py b/dendrite/browser/async_api/__init__.py similarity index 100% rename from dendrite/async_api/__init__.py rename to dendrite/browser/async_api/__init__.py diff --git a/dendrite/async_api/_api/__init__.py b/dendrite/browser/async_api/_api/__init__.py similarity index 100% rename from dendrite/async_api/_api/__init__.py rename to dendrite/browser/async_api/_api/__init__.py diff --git a/dendrite/async_api/_api/_http_client.py b/dendrite/browser/async_api/_api/_http_client.py similarity index 96% rename from dendrite/async_api/_api/_http_client.py rename to dendrite/browser/async_api/_api/_http_client.py index 9e694a6..72777e2 100644 --- a/dendrite/async_api/_api/_http_client.py +++ b/dendrite/browser/async_api/_api/_http_client.py @@ -3,8 +3,7 @@ import httpx from loguru import logger - -from dendrite.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.api_config import APIConfig class HTTPClient: diff --git a/dendrite/async_api/_api/browser_api_client.py b/dendrite/browser/async_api/_api/browser_api_client.py similarity index 71% rename from dendrite/async_api/_api/browser_api_client.py rename to dendrite/browser/async_api/_api/browser_api_client.py index 2de035d..b52738f 100644 --- a/dendrite/async_api/_api/browser_api_client.py +++ b/dendrite/browser/async_api/_api/browser_api_client.py @@ -1,31 +1,31 @@ from typing import Optional from loguru import logger -from dendrite.async_api._api.response.cache_extract_response import ( +from dendrite.browser.async_api._api.response.cache_extract_response import ( CacheExtractResponse, ) -from dendrite.async_api._api.response.selector_cache_response import ( +from dendrite.browser.async_api._api.response.selector_cache_response import ( SelectorCacheResponse, ) -from dendrite.async_api._core.models.authentication import AuthSession -from dendrite.async_api._api.response.get_element_response import GetElementResponse -from dendrite.async_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.async_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.async_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.async_api._api.dto.make_interaction_dto import MakeInteractionDTO -from dendrite.async_api._api.dto.extract_dto import ExtractDTO -from dendrite.async_api._api.dto.try_run_script_dto import TryRunScriptDTO -from dendrite.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.async_api._api.response.ask_page_response import AskPageResponse -from dendrite.async_api._api.response.interaction_response import ( +from dendrite.browser.async_api._core.models.authentication import AuthSession +from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse +from dendrite.browser.async_api._api.dto.ask_page_dto import AskPageDTO +from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO +from dendrite.browser.async_api._api.dto.get_elements_dto import GetElementsDTO +from dendrite.browser.async_api._api.dto.make_interaction_dto import MakeInteractionDTO +from dendrite.browser.async_api._api.dto.extract_dto import ExtractDTO +from dendrite.browser.async_api._api.dto.try_run_script_dto import TryRunScriptDTO +from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO +from dendrite.browser.async_api._api.response.ask_page_response import AskPageResponse +from dendrite.browser.async_api._api.response.interaction_response import ( InteractionResponse, ) -from dendrite.async_api._api.response.extract_response import ExtractResponse -from dendrite.async_api._api._http_client import HTTPClient -from dendrite._common._exceptions.dendrite_exception import ( +from dendrite.browser.async_api._api.response.extract_response import ExtractResponse +from dendrite.browser.async_api._api._http_client import HTTPClient +from dendrite.browser._common._exceptions.dendrite_exception import ( InvalidAuthSessionError, ) -from dendrite.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO +from dendrite.browser.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO class BrowserAPIClient(HTTPClient): diff --git a/dendrite/async_api/_api/dto/__init__.py b/dendrite/browser/async_api/_api/dto/__init__.py similarity index 100% rename from dendrite/async_api/_api/dto/__init__.py rename to dendrite/browser/async_api/_api/dto/__init__.py diff --git a/dendrite/sync_api/_api/dto/ask_page_dto.py b/dendrite/browser/async_api/_api/dto/ask_page_dto.py similarity index 56% rename from dendrite/sync_api/_api/dto/ask_page_dto.py rename to dendrite/browser/async_api/_api/dto/ask_page_dto.py index f3eb650..f2dcaf3 100644 --- a/dendrite/sync_api/_api/dto/ask_page_dto.py +++ b/dendrite/browser/async_api/_api/dto/ask_page_dto.py @@ -1,7 +1,7 @@ from typing import Any, Optional from pydantic import BaseModel -from dendrite.sync_api._core.models.api_config import APIConfig -from dendrite.sync_api._core.models.page_information import PageInformation +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation class AskPageDTO(BaseModel): diff --git a/dendrite/async_api/_api/dto/authenticate_dto.py b/dendrite/browser/async_api/_api/dto/authenticate_dto.py similarity index 100% rename from dendrite/async_api/_api/dto/authenticate_dto.py rename to dendrite/browser/async_api/_api/dto/authenticate_dto.py diff --git a/dendrite/async_api/_api/dto/extract_dto.py b/dendrite/browser/async_api/_api/dto/extract_dto.py similarity index 79% rename from dendrite/async_api/_api/dto/extract_dto.py rename to dendrite/browser/async_api/_api/dto/extract_dto.py index 0216cce..8cf1cc7 100644 --- a/dendrite/async_api/_api/dto/extract_dto.py +++ b/dendrite/browser/async_api/_api/dto/extract_dto.py @@ -1,8 +1,8 @@ import json from typing import Any from pydantic import BaseModel -from dendrite.async_api._core.models.api_config import APIConfig -from dendrite.async_api._core.models.page_information import PageInformation +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation class ExtractDTO(BaseModel): diff --git a/dendrite/sync_api/_api/dto/get_elements_dto.py b/dendrite/browser/async_api/_api/dto/get_elements_dto.py similarity index 70% rename from dendrite/sync_api/_api/dto/get_elements_dto.py rename to dendrite/browser/async_api/_api/dto/get_elements_dto.py index d9d2b06..636c896 100644 --- a/dendrite/sync_api/_api/dto/get_elements_dto.py +++ b/dendrite/browser/async_api/_api/dto/get_elements_dto.py @@ -1,7 +1,8 @@ from typing import Dict, Union from pydantic import BaseModel -from dendrite.sync_api._core.models.api_config import APIConfig -from dendrite.sync_api._core.models.page_information import PageInformation + +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation class CheckSelectorCacheDTO(BaseModel): diff --git a/dendrite/browser/async_api/_api/dto/get_interaction_dto.py b/dendrite/browser/async_api/_api/dto/get_interaction_dto.py new file mode 100644 index 0000000..93889c7 --- /dev/null +++ b/dendrite/browser/async_api/_api/dto/get_interaction_dto.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel + +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation + + +class GetInteractionDTO(BaseModel): + page_information: PageInformation + api_config: APIConfig + prompt: str diff --git a/dendrite/async_api/_api/dto/get_session_dto.py b/dendrite/browser/async_api/_api/dto/get_session_dto.py similarity index 100% rename from dendrite/async_api/_api/dto/get_session_dto.py rename to dendrite/browser/async_api/_api/dto/get_session_dto.py diff --git a/dendrite/async_api/_api/dto/google_search_dto.py b/dendrite/browser/async_api/_api/dto/google_search_dto.py similarity index 61% rename from dendrite/async_api/_api/dto/google_search_dto.py rename to dendrite/browser/async_api/_api/dto/google_search_dto.py index 8a16a1f..6c55615 100644 --- a/dendrite/async_api/_api/dto/google_search_dto.py +++ b/dendrite/browser/async_api/_api/dto/google_search_dto.py @@ -1,7 +1,7 @@ from typing import Optional from pydantic import BaseModel -from dendrite.async_api._core.models.api_config import APIConfig -from dendrite.async_api._core.models.page_information import PageInformation +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation class GoogleSearchDTO(BaseModel): diff --git a/dendrite/async_api/_api/dto/make_interaction_dto.py b/dendrite/browser/async_api/_api/dto/make_interaction_dto.py similarity index 72% rename from dendrite/async_api/_api/dto/make_interaction_dto.py rename to dendrite/browser/async_api/_api/dto/make_interaction_dto.py index 8edbc06..c0592ad 100644 --- a/dendrite/async_api/_api/dto/make_interaction_dto.py +++ b/dendrite/browser/async_api/_api/dto/make_interaction_dto.py @@ -1,7 +1,7 @@ from typing import Literal, Optional from pydantic import BaseModel -from dendrite.async_api._core.models.api_config import APIConfig -from dendrite.async_api._core.models.page_diff_information import ( +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_diff_information import ( PageDiffInformation, ) diff --git a/dendrite/async_api/_api/dto/try_run_script_dto.py b/dendrite/browser/async_api/_api/dto/try_run_script_dto.py similarity index 81% rename from dendrite/async_api/_api/dto/try_run_script_dto.py rename to dendrite/browser/async_api/_api/dto/try_run_script_dto.py index 2926401..e283806 100644 --- a/dendrite/async_api/_api/dto/try_run_script_dto.py +++ b/dendrite/browser/async_api/_api/dto/try_run_script_dto.py @@ -1,6 +1,6 @@ from typing import Any, Optional from pydantic import BaseModel -from dendrite.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.api_config import APIConfig class TryRunScriptDTO(BaseModel): diff --git a/dendrite/async_api/_api/dto/upload_auth_session_dto.py b/dendrite/browser/async_api/_api/dto/upload_auth_session_dto.py similarity index 71% rename from dendrite/async_api/_api/dto/upload_auth_session_dto.py rename to dendrite/browser/async_api/_api/dto/upload_auth_session_dto.py index ecb68e1..1697fdf 100644 --- a/dendrite/async_api/_api/dto/upload_auth_session_dto.py +++ b/dendrite/browser/async_api/_api/dto/upload_auth_session_dto.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from dendrite.async_api._core.models.authentication import ( +from dendrite.browser.async_api._core.models.authentication import ( AuthSession, StorageState, ) diff --git a/dendrite/async_api/_api/response/__init__.py b/dendrite/browser/async_api/_api/response/__init__.py similarity index 100% rename from dendrite/async_api/_api/response/__init__.py rename to dendrite/browser/async_api/_api/response/__init__.py diff --git a/dendrite/async_api/_api/response/ask_page_response.py b/dendrite/browser/async_api/_api/response/ask_page_response.py similarity index 100% rename from dendrite/async_api/_api/response/ask_page_response.py rename to dendrite/browser/async_api/_api/response/ask_page_response.py diff --git a/dendrite/async_api/_api/response/cache_extract_response.py b/dendrite/browser/async_api/_api/response/cache_extract_response.py similarity index 100% rename from dendrite/async_api/_api/response/cache_extract_response.py rename to dendrite/browser/async_api/_api/response/cache_extract_response.py diff --git a/dendrite/sync_api/_api/response/extract_response.py b/dendrite/browser/async_api/_api/response/extract_response.py similarity index 80% rename from dendrite/sync_api/_api/response/extract_response.py rename to dendrite/browser/async_api/_api/response/extract_response.py index 0ef6e59..dd7f6d3 100644 --- a/dendrite/sync_api/_api/response/extract_response.py +++ b/dendrite/browser/async_api/_api/response/extract_response.py @@ -1,6 +1,8 @@ from typing import Generic, Optional, TypeVar from pydantic import BaseModel -from dendrite.sync_api._common.status import Status + +from dendrite.browser.async_api._common.status import Status + T = TypeVar("T") diff --git a/dendrite/async_api/_api/response/get_element_response.py b/dendrite/browser/async_api/_api/response/get_element_response.py similarity index 80% rename from dendrite/async_api/_api/response/get_element_response.py rename to dendrite/browser/async_api/_api/response/get_element_response.py index c49caaf..8fb8fa5 100644 --- a/dendrite/async_api/_api/response/get_element_response.py +++ b/dendrite/browser/async_api/_api/response/get_element_response.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from dendrite.async_api._common.status import Status +from dendrite.browser.async_api._common.status import Status class GetElementResponse(BaseModel): diff --git a/dendrite/async_api/_api/response/google_search_response.py b/dendrite/browser/async_api/_api/response/google_search_response.py similarity index 100% rename from dendrite/async_api/_api/response/google_search_response.py rename to dendrite/browser/async_api/_api/response/google_search_response.py diff --git a/dendrite/sync_api/_api/response/interaction_response.py b/dendrite/browser/async_api/_api/response/interaction_response.py similarity index 63% rename from dendrite/sync_api/_api/response/interaction_response.py rename to dendrite/browser/async_api/_api/response/interaction_response.py index f273056..3dd2e49 100644 --- a/dendrite/sync_api/_api/response/interaction_response.py +++ b/dendrite/browser/async_api/_api/response/interaction_response.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from dendrite.sync_api._common.status import Status +from dendrite.browser.async_api._common.status import Status class InteractionResponse(BaseModel): diff --git a/dendrite/async_api/_api/response/selector_cache_response.py b/dendrite/browser/async_api/_api/response/selector_cache_response.py similarity index 100% rename from dendrite/async_api/_api/response/selector_cache_response.py rename to dendrite/browser/async_api/_api/response/selector_cache_response.py diff --git a/dendrite/async_api/_api/response/session_response.py b/dendrite/browser/async_api/_api/response/session_response.py similarity index 100% rename from dendrite/async_api/_api/response/session_response.py rename to dendrite/browser/async_api/_api/response/session_response.py diff --git a/dendrite/async_api/_common/__init__.py b/dendrite/browser/async_api/_common/__init__.py similarity index 100% rename from dendrite/async_api/_common/__init__.py rename to dendrite/browser/async_api/_common/__init__.py diff --git a/dendrite/async_api/_common/constants.py b/dendrite/browser/async_api/_common/constants.py similarity index 100% rename from dendrite/async_api/_common/constants.py rename to dendrite/browser/async_api/_common/constants.py diff --git a/dendrite/async_api/_common/event_sync.py b/dendrite/browser/async_api/_common/event_sync.py similarity index 100% rename from dendrite/async_api/_common/event_sync.py rename to dendrite/browser/async_api/_common/event_sync.py diff --git a/dendrite/async_api/_common/status.py b/dendrite/browser/async_api/_common/status.py similarity index 100% rename from dendrite/async_api/_common/status.py rename to dendrite/browser/async_api/_common/status.py diff --git a/dendrite/async_api/_core/__init__.py b/dendrite/browser/async_api/_core/__init__.py similarity index 100% rename from dendrite/async_api/_core/__init__.py rename to dendrite/browser/async_api/_core/__init__.py diff --git a/dendrite/async_api/_core/_impl_browser.py b/dendrite/browser/async_api/_core/_impl_browser.py similarity index 93% rename from dendrite/async_api/_core/_impl_browser.py rename to dendrite/browser/async_api/_core/_impl_browser.py index c4e0f99..b01f313 100644 --- a/dendrite/async_api/_core/_impl_browser.py +++ b/dendrite/browser/async_api/_core/_impl_browser.py @@ -2,9 +2,9 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_browser import AsyncDendrite + from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api._core._type_spec import PlaywrightPage from playwright.async_api import Download, Browser, Playwright diff --git a/dendrite/async_api/_core/_impl_mapping.py b/dendrite/browser/async_api/_core/_impl_mapping.py similarity index 64% rename from dendrite/async_api/_core/_impl_mapping.py rename to dendrite/browser/async_api/_core/_impl_mapping.py index 3268943..b68292a 100644 --- a/dendrite/async_api/_core/_impl_mapping.py +++ b/dendrite/browser/async_api/_core/_impl_mapping.py @@ -1,12 +1,12 @@ from typing import Any, Dict, Optional, Type -from dendrite.async_api._core._impl_browser import ImplBrowser, LocalImpl +from dendrite.browser.async_api._core._impl_browser import ImplBrowser, LocalImpl -from dendrite.async_api._ext_impl.browserbase._impl import BrowserBaseImpl -from dendrite.async_api._ext_impl.browserless._impl import BrowserlessImpl -from dendrite.remote.browserless_config import BrowserlessConfig -from dendrite.remote.browserbase_config import BrowserbaseConfig -from dendrite.remote import Providers +from dendrite.browser.async_api._ext_impl.browserbase._impl import BrowserBaseImpl +from dendrite.browser.async_api._ext_impl.browserless._impl import BrowserlessImpl +from dendrite.browser.remote.browserless_config import BrowserlessConfig +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser.remote import Providers IMPL_MAPPING: Dict[Type[Providers], Type[ImplBrowser]] = { BrowserbaseConfig: BrowserBaseImpl, diff --git a/dendrite/async_api/_core/_js/__init__.py b/dendrite/browser/async_api/_core/_js/__init__.py similarity index 100% rename from dendrite/async_api/_core/_js/__init__.py rename to dendrite/browser/async_api/_core/_js/__init__.py diff --git a/dendrite/async_api/_core/_js/eventListenerPatch.js b/dendrite/browser/async_api/_core/_js/eventListenerPatch.js similarity index 100% rename from dendrite/async_api/_core/_js/eventListenerPatch.js rename to dendrite/browser/async_api/_core/_js/eventListenerPatch.js diff --git a/dendrite/sync_api/_core/_js/generateDendriteIDs.js b/dendrite/browser/async_api/_core/_js/generateDendriteIDs.js similarity index 62% rename from dendrite/sync_api/_core/_js/generateDendriteIDs.js rename to dendrite/browser/async_api/_core/_js/generateDendriteIDs.js index 3ad8574..0ae5f61 100644 --- a/dendrite/sync_api/_core/_js/generateDendriteIDs.js +++ b/dendrite/browser/async_api/_core/_js/generateDendriteIDs.js @@ -1,55 +1,58 @@ -var hashCode = (string) => { +var hashCode = (str) => { var hash = 0, i, chr; - if (string.length === 0) return hash; - for (i = 0; i < string.length; i++) { - chr = string.charCodeAt(i); + if (str.length === 0) return hash; + for (i = 0; i < str.length; i++) { + chr = str.charCodeAt(i); hash = ((hash << 5) - hash) + chr; hash |= 0; // Convert to 32bit integer } return hash; } -var getXPathForElement = (element) => { - const getElementIndex = (element) => { - let index = 1; - let sibling = element.previousElementSibling; - - while (sibling) { - if (sibling.localName === element.localName) { - index++; - } - sibling = sibling.previousElementSibling; + +const getElementIndex = (element) => { + let index = 1; + let sibling = element.previousElementSibling; + + while (sibling) { + if (sibling.localName === element.localName) { + index++; } - - return index; - }; + sibling = sibling.previousElementSibling; + } + + return index; +}; - const segs = elm => { - if (!elm || elm.nodeType !== 1) return ['']; - if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; - const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; - let index = getElementIndex(elm); - - return [...segs(elm.parentNode), `${localName}[${index}]`]; - }; + +const segs = function elmSegs(elm) { + if (!elm || elm.nodeType !== 1) return ['']; + if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; + const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; + let index = getElementIndex(elm); + + return [...elmSegs(elm.parentNode), `${localName}[${index}]`]; +}; + +var getXPathForElement = (element) => { return segs(element).join('/'); } // Create a Map to store used hashes and their counters const usedHashes = new Map(); +var markHidden = (hidden_element) => { + // Mark the hidden element itself + hidden_element.setAttribute('data-hidden', 'true'); + +} + document.querySelectorAll('*').forEach((element, index) => { try { const xpath = getXPathForElement(element); const hash = hashCode(xpath); const baseId = hash.toString(36); - - const markHidden = (hidden_element) => { - // Mark the hidden element itself - hidden_element.setAttribute('data-hidden', 'true'); - - } // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; const isHidden = !element.checkVisibility(); diff --git a/dendrite/sync_api/_core/_js/generateDendriteIDsIframe.js b/dendrite/browser/async_api/_core/_js/generateDendriteIDsIframe.js similarity index 63% rename from dendrite/sync_api/_core/_js/generateDendriteIDsIframe.js rename to dendrite/browser/async_api/_core/_js/generateDendriteIDsIframe.js index bb2a65d..4f59cef 100644 --- a/dendrite/sync_api/_core/_js/generateDendriteIDsIframe.js +++ b/dendrite/browser/async_api/_core/_js/generateDendriteIDsIframe.js @@ -1,51 +1,54 @@ ({frame_path}) => { - var hashCode = (string) => { + var hashCode = (str) => { var hash = 0, i, chr; - if (string.length === 0) return hash; - for (i = 0; i < string.length; i++) { - chr = string.charCodeAt(i); + if (str.length === 0) return hash; + for (i = 0; i < str.length; i++) { + chr = str.charCodeAt(i); hash = ((hash << 5) - hash) + chr; hash |= 0; // Convert to 32bit integer } return hash; } - var getXPathForElement = (element) => { - const getElementIndex = (element) => { - let index = 1; - let sibling = element.previousElementSibling; - - while (sibling) { - if (sibling.localName === element.localName) { - index++; - } - sibling = sibling.previousElementSibling; + const getElementIndex = (element) => { + let index = 1; + let sibling = element.previousElementSibling; + + while (sibling) { + if (sibling.localName === element.localName) { + index++; } - - return index; - }; + sibling = sibling.previousElementSibling; + } + + return index; + }; - const segs = elm => { - if (!elm || elm.nodeType !== 1) return ['']; - if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; - const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; - let index = getElementIndex(elm); - - return [...segs(elm.parentNode), `${localName}[${index}]`]; - }; - return segs(element).join('/'); - } + const segs = function elmSegs(elm) { + if (!elm || elm.nodeType !== 1) return ['']; + if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; + const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; + let index = getElementIndex(elm); + + return [...elmSegs(elm.parentNode), `${localName}[${index}]`]; + }; + + var getXPathForElement = (element) => { + return segs(element).join('/'); + } // Create a Map to store used hashes and their counters const usedHashes = new Map(); + + var markHidden = (hidden_element) => { + // Mark the hidden element itself + hidden_element.setAttribute('data-hidden', 'true'); + } document.querySelectorAll('*').forEach((element, index) => { try { - const markHidden = (hidden_element) => { - // Mark the hidden element itself - hidden_element.setAttribute('data-hidden', 'true'); - } + // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; const isHidden = !element.checkVisibility(); @@ -57,7 +60,7 @@ }else{ element.removeAttribute("data-hidden") // in case we hid it in a previous call } - const xpath = getXPathForElement(element); + let xpath = getXPathForElement(element); if(frame_path){ element.setAttribute("iframe-path",frame_path) xpath = frame_path + xpath; diff --git a/dendrite/async_api/_core/_managers/__init__.py b/dendrite/browser/async_api/_core/_managers/__init__.py similarity index 100% rename from dendrite/async_api/_core/_managers/__init__.py rename to dendrite/browser/async_api/_core/_managers/__init__.py diff --git a/dendrite/async_api/_core/_managers/navigation_tracker.py b/dendrite/browser/async_api/_core/_managers/navigation_tracker.py similarity index 97% rename from dendrite/async_api/_core/_managers/navigation_tracker.py rename to dendrite/browser/async_api/_core/_managers/navigation_tracker.py index dc80337..71a1f05 100644 --- a/dendrite/async_api/_core/_managers/navigation_tracker.py +++ b/dendrite/browser/async_api/_core/_managers/navigation_tracker.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_page import AsyncPage + from dendrite.browser.async_api._core.dendrite_page import AsyncPage class NavigationTracker: diff --git a/dendrite/async_api/_core/_managers/page_manager.py b/dendrite/browser/async_api/_core/_managers/page_manager.py similarity index 93% rename from dendrite/async_api/_core/_managers/page_manager.py rename to dendrite/browser/async_api/_core/_managers/page_manager.py index 0d30cbf..76e5b01 100644 --- a/dendrite/async_api/_core/_managers/page_manager.py +++ b/dendrite/browser/async_api/_core/_managers/page_manager.py @@ -4,9 +4,9 @@ from playwright.async_api import BrowserContext, Download, FileChooser if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.async_api._core._type_spec import PlaywrightPage -from dendrite.async_api._core.dendrite_page import AsyncPage + from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite +from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api._core.dendrite_page import AsyncPage class PageManager: diff --git a/dendrite/async_api/_core/_managers/screenshot_manager.py b/dendrite/browser/async_api/_core/_managers/screenshot_manager.py similarity index 96% rename from dendrite/async_api/_core/_managers/screenshot_manager.py rename to dendrite/browser/async_api/_core/_managers/screenshot_manager.py index 6fce4b1..a9cceb0 100644 --- a/dendrite/async_api/_core/_managers/screenshot_manager.py +++ b/dendrite/browser/async_api/_core/_managers/screenshot_manager.py @@ -2,7 +2,7 @@ import os from uuid import uuid4 -from dendrite.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api._core._type_spec import PlaywrightPage class ScreenshotManager: diff --git a/dendrite/async_api/_core/_type_spec.py b/dendrite/browser/async_api/_core/_type_spec.py similarity index 100% rename from dendrite/async_api/_core/_type_spec.py rename to dendrite/browser/async_api/_core/_type_spec.py diff --git a/dendrite/async_api/_core/_utils.py b/dendrite/browser/async_api/_core/_utils.py similarity index 87% rename from dendrite/async_api/_core/_utils.py rename to dendrite/browser/async_api/_core/_utils.py index f030135..8860b08 100644 --- a/dendrite/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_core/_utils.py @@ -3,18 +3,18 @@ from bs4 import BeautifulSoup from loguru import logger -from dendrite.async_api._api.response.get_element_response import GetElementResponse -from dendrite.async_api._core._type_spec import PlaywrightPage -from dendrite.async_api._core.dendrite_element import AsyncElement -from dendrite.async_api._core.models.response import AsyncElementsResponse +from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse +from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api._core.dendrite_element import AsyncElement +from dendrite.browser.async_api._core.models.response import AsyncElementsResponse if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_page import AsyncPage + from dendrite.browser.async_api._core.dendrite_page import AsyncPage -from dendrite.async_api._core._js import ( +from dendrite.browser.async_api._core._js import ( GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, ) -from dendrite.async_api._dom.util.mild_strip import mild_strip_in_place +from dendrite.browser.async_api._dom.util.mild_strip import mild_strip_in_place async def expand_iframes( diff --git a/dendrite/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/_core/dendrite_browser.py similarity index 89% rename from dendrite/async_api/_core/dendrite_browser.py rename to dendrite/browser/async_api/_core/dendrite_browser.py index 07722ee..d41c251 100644 --- a/dendrite/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/_core/dendrite_browser.py @@ -15,39 +15,40 @@ FilePayload, ) -from dendrite.async_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.async_api._common.event_sync import EventSync -from dendrite.async_api._core._impl_browser import ImplBrowser -from dendrite.async_api._core._impl_mapping import get_impl -from dendrite.async_api._core._managers.page_manager import ( +from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO +from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO +from dendrite.browser.async_api._common.event_sync import EventSync +from dendrite.browser.async_api._core._impl_browser import ImplBrowser +from dendrite.browser.async_api._core._impl_mapping import get_impl +from dendrite.browser.async_api._core._managers.page_manager import ( PageManager, ) -from dendrite.async_api._core._type_spec import PlaywrightPage -from dendrite.async_api._core.dendrite_page import AsyncPage -from dendrite.async_api._common.constants import STEALTH_ARGS -from dendrite.async_api._core.mixin.ask import AskMixin -from dendrite.async_api._core.mixin.click import ClickMixin -from dendrite.async_api._core.mixin.extract import ExtractionMixin -from dendrite.async_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.async_api._core.mixin.get_element import GetElementMixin -from dendrite.async_api._core.mixin.keyboard import KeyboardMixin -from dendrite.async_api._core.mixin.screenshot import ScreenshotMixin -from dendrite.async_api._core.mixin.wait_for import WaitForMixin -from dendrite.async_api._core.mixin.markdown import MarkdownMixin -from dendrite.async_api._core.models.authentication import ( +from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api._core.dendrite_page import AsyncPage +from dendrite.browser.async_api._common.constants import STEALTH_ARGS +from dendrite.browser.async_api._core.mixin.ask import AskMixin +from dendrite.browser.async_api._core.mixin.click import ClickMixin +from dendrite.browser.async_api._core.mixin.extract import ExtractionMixin +from dendrite.browser.async_api._core.mixin.fill_fields import FillFieldsMixin +from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.async_api._core.mixin.keyboard import KeyboardMixin +from dendrite.browser.async_api._core.mixin.screenshot import ScreenshotMixin +from dendrite.browser.async_api._core.mixin.wait_for import WaitForMixin +from dendrite.browser.async_api._core.mixin.markdown import MarkdownMixin +from dendrite.browser.async_api._core.models.authentication import ( AuthSession, ) -from dendrite.async_api._core.models.api_config import APIConfig -from dendrite.async_api._api.browser_api_client import BrowserAPIClient -from dendrite._common._exceptions.dendrite_exception import ( +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._api.browser_api_client import BrowserAPIClient +from dendrite.browser._common._exceptions.dendrite_exception import ( BrowserNotLaunchedError, DendriteException, IncorrectOutcomeError, ) -from dendrite.remote import Providers +from dendrite.browser.remote import Providers +from dendrite.logic.interfaces.async_api import BrowserAPIProtocol class AsyncDendrite( @@ -131,7 +132,7 @@ def __init__( self._download_handler = EventSync(event_type=Download) self.closed = False self._auth = auth - self._browser_api_client = BrowserAPIClient(api_config, self._id) + self._browser_api_client: BrowserAPIProtocol = BrowserAPIClient(api_config, self._id) @property def pages(self) -> List[AsyncPage]: @@ -150,7 +151,7 @@ async def _get_page(self) -> AsyncPage: active_page = await self.get_active_page() return active_page - def _get_browser_api_client(self) -> BrowserAPIClient: + def _get_browser_api_client(self) -> BrowserAPIProtocol: return self._browser_api_client def _get_dendrite_browser(self) -> "AsyncDendrite": diff --git a/dendrite/async_api/_core/dendrite_element.py b/dendrite/browser/async_api/_core/dendrite_element.py similarity index 92% rename from dendrite/async_api/_core/dendrite_element.py rename to dendrite/browser/async_api/_core/dendrite_element.py index e4e4fed..6c44ecf 100644 --- a/dendrite/async_api/_core/dendrite_element.py +++ b/dendrite/browser/async_api/_core/dendrite_element.py @@ -8,20 +8,21 @@ from loguru import logger from playwright.async_api import Locator -from dendrite.async_api._api.browser_api_client import BrowserAPIClient -from dendrite._common._exceptions.dendrite_exception import IncorrectOutcomeError +from dendrite.browser.async_api._api.browser_api_client import BrowserAPIClient +from dendrite.browser._common._exceptions.dendrite_exception import IncorrectOutcomeError +from dendrite.logic.interfaces.async_api import BrowserAPIProtocol if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.async_api._core._managers.navigation_tracker import NavigationTracker -from dendrite.async_api._core.models.page_diff_information import ( + from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite +from dendrite.browser.async_api._core._managers.navigation_tracker import NavigationTracker +from dendrite.browser.async_api._core.models.page_diff_information import ( PageDiffInformation, ) -from dendrite.async_api._core._type_spec import Interaction -from dendrite.async_api._api.response.interaction_response import ( +from dendrite.browser.async_api._core._type_spec import Interaction +from dendrite.browser.async_api._api.response.interaction_response import ( InteractionResponse, ) -from dendrite.async_api._api.dto.make_interaction_dto import MakeInteractionDTO +from dendrite.browser.async_api._api.dto.make_interaction_dto import MakeInteractionDTO def perform_action(interaction_type: Interaction): @@ -108,7 +109,7 @@ def __init__( dendrite_id: str, locator: Locator, dendrite_browser: AsyncDendrite, - browser_api_client: BrowserAPIClient, + browser_api_client: BrowserAPIProtocol, ): """ Initialize a AsyncElement. diff --git a/dendrite/async_api/_core/dendrite_page.py b/dendrite/browser/async_api/_core/dendrite_page.py similarity index 89% rename from dendrite/async_api/_core/dendrite_page.py rename to dendrite/browser/async_api/_core/dendrite_page.py index 7d45eeb..13c5db3 100644 --- a/dendrite/async_api/_core/dendrite_page.py +++ b/dendrite/browser/async_api/_core/dendrite_page.py @@ -24,32 +24,33 @@ ) -from dendrite.async_api._api.browser_api_client import BrowserAPIClient -from dendrite.async_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT -from dendrite.async_api._core._type_spec import PlaywrightPage -from dendrite.async_api._core.dendrite_element import AsyncElement -from dendrite.async_api._core.mixin.ask import AskMixin -from dendrite.async_api._core.mixin.click import ClickMixin -from dendrite.async_api._core.mixin.extract import ExtractionMixin -from dendrite.async_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.async_api._core.mixin.get_element import GetElementMixin -from dendrite.async_api._core.mixin.keyboard import KeyboardMixin -from dendrite.async_api._core.mixin.markdown import MarkdownMixin -from dendrite.async_api._core.mixin.wait_for import WaitForMixin -from dendrite.async_api._core.models.page_information import PageInformation +from dendrite.browser.async_api._api.browser_api_client import BrowserAPIClient +from dendrite.browser.async_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT +from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api._core.dendrite_element import AsyncElement +from dendrite.browser.async_api._core.mixin.ask import AskMixin +from dendrite.browser.async_api._core.mixin.click import ClickMixin +from dendrite.browser.async_api._core.mixin.extract import ExtractionMixin +from dendrite.browser.async_api._core.mixin.fill_fields import FillFieldsMixin +from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.async_api._core.mixin.keyboard import KeyboardMixin +from dendrite.browser.async_api._core.mixin.markdown import MarkdownMixin +from dendrite.browser.async_api._core.mixin.wait_for import WaitForMixin +from dendrite.browser.async_api._core.models.page_information import PageInformation +from dendrite.logic.interfaces.async_api import BrowserAPIProtocol if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_browser import AsyncDendrite + from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.async_api._core._managers.screenshot_manager import ScreenshotManager -from dendrite._common._exceptions.dendrite_exception import ( +from dendrite.browser.async_api._core._managers.screenshot_manager import ScreenshotManager +from dendrite.browser._common._exceptions.dendrite_exception import ( DendriteException, ) -from dendrite.async_api._core._utils import ( +from dendrite.browser.async_api._core._utils import ( expand_iframes, ) @@ -75,7 +76,7 @@ def __init__( self, page: PlaywrightPage, dendrite_browser: "AsyncDendrite", - browser_api_client: "BrowserAPIClient", + browser_api_client: "BrowserAPIProtocol", ): self.playwright_page = page self.screenshot_manager = ScreenshotManager(page) @@ -117,7 +118,7 @@ async def _get_page(self) -> "AsyncPage": def _get_dendrite_browser(self) -> "AsyncDendrite": return self.dendrite_browser - def _get_browser_api_client(self) -> BrowserAPIClient: + def _get_browser_api_client(self) -> BrowserAPIProtocol: return self._browser_api_client async def goto( @@ -292,7 +293,7 @@ async def _generate_dendrite_ids(self): await self.playwright_page.wait_for_load_state( state="load", timeout=3000 ) - logger.debug( + logger.exception( f"Failed to generate dendrite IDs: {e}, attempt {tries+1}/3" ) tries += 1 diff --git a/dendrite/async_api/_core/mixin/ask.py b/dendrite/browser/async_api/_core/mixin/ask.py similarity index 96% rename from dendrite/async_api/_core/mixin/ask.py rename to dendrite/browser/async_api/_core/mixin/ask.py index 05f6a04..79f9401 100644 --- a/dendrite/async_api/_core/mixin/ask.py +++ b/dendrite/browser/async_api/_core/mixin/ask.py @@ -4,16 +4,16 @@ from loguru import logger -from dendrite.async_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.async_api._core._type_spec import ( +from dendrite.browser.async_api._api.dto.ask_page_dto import AskPageDTO +from dendrite.browser.async_api._core._type_spec import ( JsonSchema, PydanticModel, TypeSpec, convert_to_type_spec, to_json_schema, ) -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException # The timeout interval between retries in milliseconds TIMEOUT_INTERVAL = [150, 450, 1000] diff --git a/dendrite/async_api/_core/mixin/click.py b/dendrite/browser/async_api/_core/mixin/click.py similarity index 86% rename from dendrite/async_api/_core/mixin/click.py rename to dendrite/browser/async_api/_core/mixin/click.py index e8b0370..8dac206 100644 --- a/dendrite/async_api/_core/mixin/click.py +++ b/dendrite/browser/async_api/_core/mixin/click.py @@ -1,11 +1,11 @@ import asyncio from typing import Any, Optional -from dendrite.async_api._api.response.interaction_response import ( +from dendrite.browser.async_api._api.response.interaction_response import ( InteractionResponse, ) -from dendrite.async_api._core.mixin.get_element import GetElementMixin -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException class ClickMixin(GetElementMixin, DendritePageProtocol): diff --git a/dendrite/async_api/_core/mixin/extract.py b/dendrite/browser/async_api/_core/mixin/extract.py similarity index 94% rename from dendrite/async_api/_core/mixin/extract.py rename to dendrite/browser/async_api/_core/mixin/extract.py index a0c4347..55d756c 100644 --- a/dendrite/async_api/_core/mixin/extract.py +++ b/dendrite/browser/async_api/_core/mixin/extract.py @@ -1,20 +1,20 @@ import asyncio import time from typing import Any, Optional, Type, overload, List -from dendrite.async_api._api.dto.extract_dto import ExtractDTO -from dendrite.async_api._api.response.cache_extract_response import ( +from dendrite.browser.async_api._api.dto.extract_dto import ExtractDTO +from dendrite.browser.async_api._api.response.cache_extract_response import ( CacheExtractResponse, ) -from dendrite.async_api._api.response.extract_response import ExtractResponse -from dendrite.async_api._core._type_spec import ( +from dendrite.browser.async_api._api.response.extract_response import ExtractResponse +from dendrite.browser.async_api._core._type_spec import ( JsonSchema, PydanticModel, TypeSpec, convert_to_type_spec, to_json_schema, ) -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.async_api._core._managers.navigation_tracker import NavigationTracker +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.async_api._core._managers.navigation_tracker import NavigationTracker from loguru import logger diff --git a/dendrite/async_api/_core/mixin/fill_fields.py b/dendrite/browser/async_api/_core/mixin/fill_fields.py similarity index 90% rename from dendrite/async_api/_core/mixin/fill_fields.py rename to dendrite/browser/async_api/_core/mixin/fill_fields.py index 55d5760..92ed5a4 100644 --- a/dendrite/async_api/_core/mixin/fill_fields.py +++ b/dendrite/browser/async_api/_core/mixin/fill_fields.py @@ -1,11 +1,11 @@ import asyncio from typing import Any, Dict, Optional -from dendrite.async_api._api.response.interaction_response import ( +from dendrite.browser.async_api._api.response.interaction_response import ( InteractionResponse, ) -from dendrite.async_api._core.mixin.get_element import GetElementMixin -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException class FillFieldsMixin(GetElementMixin, DendritePageProtocol): diff --git a/dendrite/async_api/_core/mixin/get_element.py b/dendrite/browser/async_api/_core/mixin/get_element.py similarity index 95% rename from dendrite/async_api/_core/mixin/get_element.py rename to dendrite/browser/async_api/_core/mixin/get_element.py index 4d54f67..fd5da2a 100644 --- a/dendrite/async_api/_core/mixin/get_element.py +++ b/dendrite/browser/async_api/_core/mixin/get_element.py @@ -4,14 +4,14 @@ from loguru import logger -from dendrite.async_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.async_api._api.response.get_element_response import GetElementResponse -from dendrite.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO -from dendrite.async_api._core._utils import get_elements_from_selectors_soup -from dendrite.async_api._core.dendrite_element import AsyncElement -from dendrite.async_api._core.models.response import AsyncElementsResponse -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._api.dto.get_elements_dto import GetElementsDTO +from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse +from dendrite.browser.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO +from dendrite.browser.async_api._core._utils import get_elements_from_selectors_soup +from dendrite.browser.async_api._core.dendrite_element import AsyncElement +from dendrite.browser.async_api._core.models.response import AsyncElementsResponse +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.async_api._core.models.api_config import APIConfig CACHE_TIMEOUT = 5 diff --git a/dendrite/async_api/_core/mixin/keyboard.py b/dendrite/browser/async_api/_core/mixin/keyboard.py similarity index 91% rename from dendrite/async_api/_core/mixin/keyboard.py rename to dendrite/browser/async_api/_core/mixin/keyboard.py index ee26559..8250370 100644 --- a/dendrite/async_api/_core/mixin/keyboard.py +++ b/dendrite/browser/async_api/_core/mixin/keyboard.py @@ -1,6 +1,6 @@ from typing import Any, Union, Literal -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException class KeyboardMixin(DendritePageProtocol): diff --git a/dendrite/async_api/_core/mixin/markdown.py b/dendrite/browser/async_api/_core/mixin/markdown.py similarity index 89% rename from dendrite/async_api/_core/mixin/markdown.py rename to dendrite/browser/async_api/_core/mixin/markdown.py index 01ada25..68e8adc 100644 --- a/dendrite/async_api/_core/mixin/markdown.py +++ b/dendrite/browser/async_api/_core/mixin/markdown.py @@ -2,8 +2,8 @@ from bs4 import BeautifulSoup import re -from dendrite.async_api._core.mixin.extract import ExtractionMixin -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.async_api._core.mixin.extract import ExtractionMixin +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol from markdownify import markdownify as md diff --git a/dendrite/async_api/_core/mixin/screenshot.py b/dendrite/browser/async_api/_core/mixin/screenshot.py similarity index 87% rename from dendrite/async_api/_core/mixin/screenshot.py rename to dendrite/browser/async_api/_core/mixin/screenshot.py index c150eb4..9877319 100644 --- a/dendrite/async_api/_core/mixin/screenshot.py +++ b/dendrite/browser/async_api/_core/mixin/screenshot.py @@ -1,4 +1,4 @@ -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol class ScreenshotMixin(DendritePageProtocol): diff --git a/dendrite/async_api/_core/mixin/wait_for.py b/dendrite/browser/async_api/_core/mixin/wait_for.py similarity index 87% rename from dendrite/async_api/_core/mixin/wait_for.py rename to dendrite/browser/async_api/_core/mixin/wait_for.py index 6bd042e..1a2b029 100644 --- a/dendrite/async_api/_core/mixin/wait_for.py +++ b/dendrite/browser/async_api/_core/mixin/wait_for.py @@ -2,10 +2,10 @@ import time -from dendrite.async_api._core.mixin.ask import AskMixin -from dendrite.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import PageConditionNotMet -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.async_api._core.mixin.ask import AskMixin +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import PageConditionNotMet +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException from loguru import logger diff --git a/dendrite/async_api/_core/models/__init__.py b/dendrite/browser/async_api/_core/models/__init__.py similarity index 100% rename from dendrite/async_api/_core/models/__init__.py rename to dendrite/browser/async_api/_core/models/__init__.py diff --git a/dendrite/async_api/_core/models/api_config.py b/dendrite/browser/async_api/_core/models/api_config.py similarity index 93% rename from dendrite/async_api/_core/models/api_config.py rename to dendrite/browser/async_api/_core/models/api_config.py index fd92cac..43f652a 100644 --- a/dendrite/async_api/_core/models/api_config.py +++ b/dendrite/browser/async_api/_core/models/api_config.py @@ -1,7 +1,7 @@ from typing import Optional from pydantic import BaseModel, model_validator -from dendrite._common._exceptions.dendrite_exception import MissingApiKeyError +from dendrite.browser._common._exceptions.dendrite_exception import MissingApiKeyError class APIConfig(BaseModel): diff --git a/dendrite/async_api/_core/models/authentication.py b/dendrite/browser/async_api/_core/models/authentication.py similarity index 100% rename from dendrite/async_api/_core/models/authentication.py rename to dendrite/browser/async_api/_core/models/authentication.py diff --git a/dendrite/async_api/_core/models/download_interface.py b/dendrite/browser/async_api/_core/models/download_interface.py similarity index 100% rename from dendrite/async_api/_core/models/download_interface.py rename to dendrite/browser/async_api/_core/models/download_interface.py diff --git a/dendrite/async_api/_core/models/page_diff_information.py b/dendrite/browser/async_api/_core/models/page_diff_information.py similarity index 61% rename from dendrite/async_api/_core/models/page_diff_information.py rename to dendrite/browser/async_api/_core/models/page_diff_information.py index 786bbc3..dd37e9f 100644 --- a/dendrite/async_api/_core/models/page_diff_information.py +++ b/dendrite/browser/async_api/_core/models/page_diff_information.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from dendrite.async_api._core.models.page_information import PageInformation +from dendrite.browser.async_api._core.models.page_information import PageInformation class PageDiffInformation(BaseModel): diff --git a/dendrite/async_api/_core/models/page_information.py b/dendrite/browser/async_api/_core/models/page_information.py similarity index 100% rename from dendrite/async_api/_core/models/page_information.py rename to dendrite/browser/async_api/_core/models/page_information.py diff --git a/dendrite/async_api/_core/models/response.py b/dendrite/browser/async_api/_core/models/response.py similarity index 96% rename from dendrite/async_api/_core/models/response.py rename to dendrite/browser/async_api/_core/models/response.py index 79b216f..ba2c917 100644 --- a/dendrite/async_api/_core/models/response.py +++ b/dendrite/browser/async_api/_core/models/response.py @@ -1,6 +1,6 @@ from typing import Dict, Iterator -from dendrite.async_api._core.dendrite_element import AsyncElement +from dendrite.browser.async_api._core.dendrite_element import AsyncElement class AsyncElementsResponse: diff --git a/dendrite/async_api/_core/protocol/page_protocol.py b/dendrite/browser/async_api/_core/protocol/page_protocol.py similarity index 63% rename from dendrite/async_api/_core/protocol/page_protocol.py rename to dendrite/browser/async_api/_core/protocol/page_protocol.py index 2aa9449..c927a08 100644 --- a/dendrite/async_api/_core/protocol/page_protocol.py +++ b/dendrite/browser/async_api/_core/protocol/page_protocol.py @@ -1,10 +1,10 @@ from typing import TYPE_CHECKING, Protocol -from dendrite.async_api._api.browser_api_client import BrowserAPIClient +from dendrite.browser.async_api._api.browser_api_client import BrowserAPIClient if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_page import AsyncPage - from dendrite.async_api._core.dendrite_browser import AsyncDendrite + from dendrite.browser.async_api._core.dendrite_page import AsyncPage + from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite class DendritePageProtocol(Protocol): diff --git a/dendrite/async_api/_dom/__init__.py b/dendrite/browser/async_api/_dom/__init__.py similarity index 100% rename from dendrite/async_api/_dom/__init__.py rename to dendrite/browser/async_api/_dom/__init__.py diff --git a/dendrite/async_api/_dom/util/mild_strip.py b/dendrite/browser/async_api/_dom/util/mild_strip.py similarity index 100% rename from dendrite/async_api/_dom/util/mild_strip.py rename to dendrite/browser/async_api/_dom/util/mild_strip.py diff --git a/dendrite/async_api/_ext_impl/__init__.py b/dendrite/browser/async_api/_ext_impl/__init__.py similarity index 100% rename from dendrite/async_api/_ext_impl/__init__.py rename to dendrite/browser/async_api/_ext_impl/__init__.py diff --git a/dendrite/async_api/_ext_impl/browserbase/__init__.py b/dendrite/browser/async_api/_ext_impl/browserbase/__init__.py similarity index 100% rename from dendrite/async_api/_ext_impl/browserbase/__init__.py rename to dendrite/browser/async_api/_ext_impl/browserbase/__init__.py diff --git a/dendrite/async_api/_ext_impl/browserbase/_client.py b/dendrite/browser/async_api/_ext_impl/browserbase/_client.py similarity index 97% rename from dendrite/async_api/_ext_impl/browserbase/_client.py rename to dendrite/browser/async_api/_ext_impl/browserbase/_client.py index 0689641..79b7dce 100644 --- a/dendrite/async_api/_ext_impl/browserbase/_client.py +++ b/dendrite/browser/async_api/_ext_impl/browserbase/_client.py @@ -5,7 +5,7 @@ import httpx from loguru import logger -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException class BrowserbaseClient: diff --git a/dendrite/async_api/_ext_impl/browserbase/_download.py b/dendrite/browser/async_api/_ext_impl/browserbase/_download.py similarity index 92% rename from dendrite/async_api/_ext_impl/browserbase/_download.py rename to dendrite/browser/async_api/_ext_impl/browserbase/_download.py index d18561c..dc355c3 100644 --- a/dendrite/async_api/_ext_impl/browserbase/_download.py +++ b/dendrite/browser/async_api/_ext_impl/browserbase/_download.py @@ -6,8 +6,8 @@ from loguru import logger from playwright.async_api import Download -from dendrite.async_api._core.models.download_interface import DownloadInterface -from dendrite.async_api._ext_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.async_api._core.models.download_interface import DownloadInterface +from dendrite.browser.async_api._ext_impl.browserbase._client import BrowserbaseClient class AsyncBrowserbaseDownload(DownloadInterface): diff --git a/dendrite/async_api/_ext_impl/browserbase/_impl.py b/dendrite/browser/async_api/_ext_impl/browserbase/_impl.py similarity index 80% rename from dendrite/async_api/_ext_impl/browserbase/_impl.py rename to dendrite/browser/async_api/_ext_impl/browserbase/_impl.py index c67846e..d8eb1be 100644 --- a/dendrite/async_api/_ext_impl/browserbase/_impl.py +++ b/dendrite/browser/async_api/_ext_impl/browserbase/_impl.py @@ -1,16 +1,16 @@ from typing import TYPE_CHECKING, Optional -from dendrite._common._exceptions.dendrite_exception import BrowserNotLaunchedError -from dendrite.async_api._core._impl_browser import ImplBrowser -from dendrite.async_api._core._type_spec import PlaywrightPage -from dendrite.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser._common._exceptions.dendrite_exception import BrowserNotLaunchedError +from dendrite.browser.async_api._core._impl_browser import ImplBrowser +from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.async_api._ext_impl.browserbase._client import BrowserbaseClient + from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite +from dendrite.browser.async_api._ext_impl.browserbase._client import BrowserbaseClient from playwright.async_api import Playwright from loguru import logger -from dendrite.async_api._ext_impl.browserbase._download import ( +from dendrite.browser.async_api._ext_impl.browserbase._download import ( AsyncBrowserbaseDownload, ) diff --git a/dendrite/async_api/_ext_impl/browserless/__init__.py b/dendrite/browser/async_api/_ext_impl/browserless/__init__.py similarity index 100% rename from dendrite/async_api/_ext_impl/browserless/__init__.py rename to dendrite/browser/async_api/_ext_impl/browserless/__init__.py diff --git a/dendrite/async_api/_ext_impl/browserless/_impl.py b/dendrite/browser/async_api/_ext_impl/browserless/_impl.py similarity index 74% rename from dendrite/async_api/_ext_impl/browserless/_impl.py rename to dendrite/browser/async_api/_ext_impl/browserless/_impl.py index e5b87b4..1ee4eb7 100644 --- a/dendrite/async_api/_ext_impl/browserless/_impl.py +++ b/dendrite/browser/async_api/_ext_impl/browserless/_impl.py @@ -1,18 +1,18 @@ import json from typing import TYPE_CHECKING, Optional -from dendrite._common._exceptions.dendrite_exception import BrowserNotLaunchedError -from dendrite.async_api._core._impl_browser import ImplBrowser -from dendrite.async_api._core._type_spec import PlaywrightPage -from dendrite.remote.browserless_config import BrowserlessConfig +from dendrite.browser._common._exceptions.dendrite_exception import BrowserNotLaunchedError +from dendrite.browser.async_api._core._impl_browser import ImplBrowser +from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.remote.browserless_config import BrowserlessConfig if TYPE_CHECKING: - from dendrite.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.async_api._ext_impl.browserbase._client import BrowserbaseClient + from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite +from dendrite.browser.async_api._ext_impl.browserbase._client import BrowserbaseClient from playwright.async_api import Playwright from loguru import logger import urllib.parse -from dendrite.async_api._ext_impl.browserbase._download import ( +from dendrite.browser.async_api._ext_impl.browserbase._download import ( AsyncBrowserbaseDownload, ) diff --git a/dendrite/browser/remote/__init__.py b/dendrite/browser/remote/__init__.py new file mode 100644 index 0000000..d5c785b --- /dev/null +++ b/dendrite/browser/remote/__init__.py @@ -0,0 +1,8 @@ +from typing import Union +from dendrite.browser.remote.browserless_config import BrowserlessConfig +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig + + +Providers = Union[BrowserbaseConfig, BrowserlessConfig] + +__all__ = ["Providers", "BrowserbaseConfig"] diff --git a/dendrite/remote/browserbase_config.py b/dendrite/browser/remote/browserbase_config.py similarity index 100% rename from dendrite/remote/browserbase_config.py rename to dendrite/browser/remote/browserbase_config.py diff --git a/dendrite/remote/browserless_config.py b/dendrite/browser/remote/browserless_config.py similarity index 88% rename from dendrite/remote/browserless_config.py rename to dendrite/browser/remote/browserless_config.py index 88a4efe..7e3bcc8 100644 --- a/dendrite/remote/browserless_config.py +++ b/dendrite/browser/remote/browserless_config.py @@ -1,7 +1,7 @@ import os from typing import Optional -from dendrite._common._exceptions.dendrite_exception import MissingApiKeyError +from dendrite.browser._common._exceptions.dendrite_exception import MissingApiKeyError class BrowserlessConfig: diff --git a/dendrite/remote/provider.py b/dendrite/browser/remote/provider.py similarity index 92% rename from dendrite/remote/provider.py rename to dendrite/browser/remote/provider.py index 8a5135f..87890a9 100644 --- a/dendrite/remote/provider.py +++ b/dendrite/browser/remote/provider.py @@ -2,8 +2,8 @@ from typing import Union -from dendrite.remote import Providers -from dendrite.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser.remote import Providers +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig try: diff --git a/dendrite/sync_api/__init__.py b/dendrite/browser/sync_api/__init__.py similarity index 100% rename from dendrite/sync_api/__init__.py rename to dendrite/browser/sync_api/__init__.py diff --git a/dendrite/sync_api/_api/__init__.py b/dendrite/browser/sync_api/_api/__init__.py similarity index 100% rename from dendrite/sync_api/_api/__init__.py rename to dendrite/browser/sync_api/_api/__init__.py diff --git a/dendrite/sync_api/_api/_http_client.py b/dendrite/browser/sync_api/_api/_http_client.py similarity index 96% rename from dendrite/sync_api/_api/_http_client.py rename to dendrite/browser/sync_api/_api/_http_client.py index e80ab64..2d053d8 100644 --- a/dendrite/sync_api/_api/_http_client.py +++ b/dendrite/browser/sync_api/_api/_http_client.py @@ -2,7 +2,7 @@ from typing import Optional import httpx from loguru import logger -from dendrite.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._core.models.api_config import APIConfig class HTTPClient: diff --git a/dendrite/sync_api/_api/browser_api_client.py b/dendrite/browser/sync_api/_api/browser_api_client.py similarity index 69% rename from dendrite/sync_api/_api/browser_api_client.py rename to dendrite/browser/sync_api/_api/browser_api_client.py index 54da703..c3ce4ab 100644 --- a/dendrite/sync_api/_api/browser_api_client.py +++ b/dendrite/browser/sync_api/_api/browser_api_client.py @@ -1,24 +1,24 @@ from typing import Optional from loguru import logger -from dendrite.sync_api._api.response.cache_extract_response import CacheExtractResponse -from dendrite.sync_api._api.response.selector_cache_response import ( +from dendrite.browser.sync_api._api.response.cache_extract_response import CacheExtractResponse +from dendrite.browser.sync_api._api.response.selector_cache_response import ( SelectorCacheResponse, ) -from dendrite.sync_api._core.models.authentication import AuthSession -from dendrite.sync_api._api.response.get_element_response import GetElementResponse -from dendrite.sync_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.sync_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.sync_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.sync_api._api.dto.make_interaction_dto import MakeInteractionDTO -from dendrite.sync_api._api.dto.extract_dto import ExtractDTO -from dendrite.sync_api._api.dto.try_run_script_dto import TryRunScriptDTO -from dendrite.sync_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.sync_api._api.response.ask_page_response import AskPageResponse -from dendrite.sync_api._api.response.interaction_response import InteractionResponse -from dendrite.sync_api._api.response.extract_response import ExtractResponse -from dendrite.sync_api._api._http_client import HTTPClient -from dendrite._common._exceptions.dendrite_exception import InvalidAuthSessionError -from dendrite.sync_api._api.dto.get_elements_dto import CheckSelectorCacheDTO +from dendrite.browser.sync_api._core.models.authentication import AuthSession +from dendrite.browser.sync_api._api.response.get_element_response import GetElementResponse +from dendrite.browser.sync_api._api.dto.ask_page_dto import AskPageDTO +from dendrite.browser.sync_api._api.dto.authenticate_dto import AuthenticateDTO +from dendrite.browser.sync_api._api.dto.get_elements_dto import GetElementsDTO +from dendrite.browser.sync_api._api.dto.make_interaction_dto import MakeInteractionDTO +from dendrite.browser.sync_api._api.dto.extract_dto import ExtractDTO +from dendrite.browser.sync_api._api.dto.try_run_script_dto import TryRunScriptDTO +from dendrite.browser.sync_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO +from dendrite.browser.sync_api._api.response.ask_page_response import AskPageResponse +from dendrite.browser.sync_api._api.response.interaction_response import InteractionResponse +from dendrite.browser.sync_api._api.response.extract_response import ExtractResponse +from dendrite.browser.sync_api._api._http_client import HTTPClient +from dendrite.browser._common._exceptions.dendrite_exception import InvalidAuthSessionError +from dendrite.browser.sync_api._api.dto.get_elements_dto import CheckSelectorCacheDTO class BrowserAPIClient(HTTPClient): diff --git a/dendrite/sync_api/_api/dto/__init__.py b/dendrite/browser/sync_api/_api/dto/__init__.py similarity index 100% rename from dendrite/sync_api/_api/dto/__init__.py rename to dendrite/browser/sync_api/_api/dto/__init__.py diff --git a/dendrite/async_api/_api/dto/ask_page_dto.py b/dendrite/browser/sync_api/_api/dto/ask_page_dto.py similarity index 57% rename from dendrite/async_api/_api/dto/ask_page_dto.py rename to dendrite/browser/sync_api/_api/dto/ask_page_dto.py index 770d172..5b068e1 100644 --- a/dendrite/async_api/_api/dto/ask_page_dto.py +++ b/dendrite/browser/sync_api/_api/dto/ask_page_dto.py @@ -1,7 +1,7 @@ from typing import Any, Optional from pydantic import BaseModel -from dendrite.async_api._core.models.api_config import APIConfig -from dendrite.async_api._core.models.page_information import PageInformation +from dendrite.browser.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._core.models.page_information import PageInformation class AskPageDTO(BaseModel): diff --git a/dendrite/sync_api/_api/dto/authenticate_dto.py b/dendrite/browser/sync_api/_api/dto/authenticate_dto.py similarity index 100% rename from dendrite/sync_api/_api/dto/authenticate_dto.py rename to dendrite/browser/sync_api/_api/dto/authenticate_dto.py diff --git a/dendrite/sync_api/_api/dto/extract_dto.py b/dendrite/browser/sync_api/_api/dto/extract_dto.py similarity index 79% rename from dendrite/sync_api/_api/dto/extract_dto.py rename to dendrite/browser/sync_api/_api/dto/extract_dto.py index f2f7694..01fccfc 100644 --- a/dendrite/sync_api/_api/dto/extract_dto.py +++ b/dendrite/browser/sync_api/_api/dto/extract_dto.py @@ -1,8 +1,8 @@ import json from typing import Any from pydantic import BaseModel -from dendrite.sync_api._core.models.api_config import APIConfig -from dendrite.sync_api._core.models.page_information import PageInformation +from dendrite.browser.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._core.models.page_information import PageInformation class ExtractDTO(BaseModel): diff --git a/dendrite/async_api/_api/dto/get_elements_dto.py b/dendrite/browser/sync_api/_api/dto/get_elements_dto.py similarity index 70% rename from dendrite/async_api/_api/dto/get_elements_dto.py rename to dendrite/browser/sync_api/_api/dto/get_elements_dto.py index 86118a2..8fe6a88 100644 --- a/dendrite/async_api/_api/dto/get_elements_dto.py +++ b/dendrite/browser/sync_api/_api/dto/get_elements_dto.py @@ -1,8 +1,7 @@ from typing import Dict, Union from pydantic import BaseModel - -from dendrite.async_api._core.models.api_config import APIConfig -from dendrite.async_api._core.models.page_information import PageInformation +from dendrite.browser.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._core.models.page_information import PageInformation class CheckSelectorCacheDTO(BaseModel): diff --git a/dendrite/browser/sync_api/_api/dto/get_interaction_dto.py b/dendrite/browser/sync_api/_api/dto/get_interaction_dto.py new file mode 100644 index 0000000..bb04c19 --- /dev/null +++ b/dendrite/browser/sync_api/_api/dto/get_interaction_dto.py @@ -0,0 +1,9 @@ +from pydantic import BaseModel +from dendrite.browser.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._core.models.page_information import PageInformation + + +class GetInteractionDTO(BaseModel): + page_information: PageInformation + api_config: APIConfig + prompt: str diff --git a/dendrite/sync_api/_api/dto/get_session_dto.py b/dendrite/browser/sync_api/_api/dto/get_session_dto.py similarity index 100% rename from dendrite/sync_api/_api/dto/get_session_dto.py rename to dendrite/browser/sync_api/_api/dto/get_session_dto.py diff --git a/dendrite/sync_api/_api/dto/google_search_dto.py b/dendrite/browser/sync_api/_api/dto/google_search_dto.py similarity index 62% rename from dendrite/sync_api/_api/dto/google_search_dto.py rename to dendrite/browser/sync_api/_api/dto/google_search_dto.py index 3e81d7d..b1d7fd4 100644 --- a/dendrite/sync_api/_api/dto/google_search_dto.py +++ b/dendrite/browser/sync_api/_api/dto/google_search_dto.py @@ -1,7 +1,7 @@ from typing import Optional from pydantic import BaseModel -from dendrite.sync_api._core.models.api_config import APIConfig -from dendrite.sync_api._core.models.page_information import PageInformation +from dendrite.browser.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._core.models.page_information import PageInformation class GoogleSearchDTO(BaseModel): diff --git a/dendrite/sync_api/_api/dto/make_interaction_dto.py b/dendrite/browser/sync_api/_api/dto/make_interaction_dto.py similarity index 69% rename from dendrite/sync_api/_api/dto/make_interaction_dto.py rename to dendrite/browser/sync_api/_api/dto/make_interaction_dto.py index 2d806a3..b2c3741 100644 --- a/dendrite/sync_api/_api/dto/make_interaction_dto.py +++ b/dendrite/browser/sync_api/_api/dto/make_interaction_dto.py @@ -1,7 +1,7 @@ from typing import Literal, Optional from pydantic import BaseModel -from dendrite.sync_api._core.models.api_config import APIConfig -from dendrite.sync_api._core.models.page_diff_information import PageDiffInformation +from dendrite.browser.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._core.models.page_diff_information import PageDiffInformation InteractionType = Literal["click", "fill", "hover"] diff --git a/dendrite/sync_api/_api/dto/try_run_script_dto.py b/dendrite/browser/sync_api/_api/dto/try_run_script_dto.py similarity index 77% rename from dendrite/sync_api/_api/dto/try_run_script_dto.py rename to dendrite/browser/sync_api/_api/dto/try_run_script_dto.py index 778c251..a2b99ea 100644 --- a/dendrite/sync_api/_api/dto/try_run_script_dto.py +++ b/dendrite/browser/sync_api/_api/dto/try_run_script_dto.py @@ -1,6 +1,6 @@ from typing import Any, Optional from pydantic import BaseModel -from dendrite.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._core.models.api_config import APIConfig class TryRunScriptDTO(BaseModel): diff --git a/dendrite/sync_api/_api/dto/upload_auth_session_dto.py b/dendrite/browser/sync_api/_api/dto/upload_auth_session_dto.py similarity index 58% rename from dendrite/sync_api/_api/dto/upload_auth_session_dto.py rename to dendrite/browser/sync_api/_api/dto/upload_auth_session_dto.py index 0741b65..72e5535 100644 --- a/dendrite/sync_api/_api/dto/upload_auth_session_dto.py +++ b/dendrite/browser/sync_api/_api/dto/upload_auth_session_dto.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from dendrite.sync_api._core.models.authentication import AuthSession, StorageState +from dendrite.browser.sync_api._core.models.authentication import AuthSession, StorageState class UploadAuthSessionDTO(BaseModel): diff --git a/dendrite/sync_api/_api/response/__init__.py b/dendrite/browser/sync_api/_api/response/__init__.py similarity index 100% rename from dendrite/sync_api/_api/response/__init__.py rename to dendrite/browser/sync_api/_api/response/__init__.py diff --git a/dendrite/sync_api/_api/response/ask_page_response.py b/dendrite/browser/sync_api/_api/response/ask_page_response.py similarity index 100% rename from dendrite/sync_api/_api/response/ask_page_response.py rename to dendrite/browser/sync_api/_api/response/ask_page_response.py diff --git a/dendrite/sync_api/_api/response/cache_extract_response.py b/dendrite/browser/sync_api/_api/response/cache_extract_response.py similarity index 100% rename from dendrite/sync_api/_api/response/cache_extract_response.py rename to dendrite/browser/sync_api/_api/response/cache_extract_response.py diff --git a/dendrite/async_api/_api/response/extract_response.py b/dendrite/browser/sync_api/_api/response/extract_response.py similarity index 81% rename from dendrite/async_api/_api/response/extract_response.py rename to dendrite/browser/sync_api/_api/response/extract_response.py index ffc0e34..7cfbbb2 100644 --- a/dendrite/async_api/_api/response/extract_response.py +++ b/dendrite/browser/sync_api/_api/response/extract_response.py @@ -1,8 +1,6 @@ from typing import Generic, Optional, TypeVar from pydantic import BaseModel - -from dendrite.async_api._common.status import Status - +from dendrite.browser.sync_api._common.status import Status T = TypeVar("T") diff --git a/dendrite/sync_api/_api/response/get_element_response.py b/dendrite/browser/sync_api/_api/response/get_element_response.py similarity index 81% rename from dendrite/sync_api/_api/response/get_element_response.py rename to dendrite/browser/sync_api/_api/response/get_element_response.py index d268cca..9401e09 100644 --- a/dendrite/sync_api/_api/response/get_element_response.py +++ b/dendrite/browser/sync_api/_api/response/get_element_response.py @@ -1,6 +1,6 @@ from typing import Dict, List, Optional, Union from pydantic import BaseModel -from dendrite.sync_api._common.status import Status +from dendrite.browser.sync_api._common.status import Status class GetElementResponse(BaseModel): diff --git a/dendrite/sync_api/_api/response/google_search_response.py b/dendrite/browser/sync_api/_api/response/google_search_response.py similarity index 100% rename from dendrite/sync_api/_api/response/google_search_response.py rename to dendrite/browser/sync_api/_api/response/google_search_response.py diff --git a/dendrite/async_api/_api/response/interaction_response.py b/dendrite/browser/sync_api/_api/response/interaction_response.py similarity index 64% rename from dendrite/async_api/_api/response/interaction_response.py rename to dendrite/browser/sync_api/_api/response/interaction_response.py index 3d24a6a..6d85f96 100644 --- a/dendrite/async_api/_api/response/interaction_response.py +++ b/dendrite/browser/sync_api/_api/response/interaction_response.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from dendrite.async_api._common.status import Status +from dendrite.browser.sync_api._common.status import Status class InteractionResponse(BaseModel): diff --git a/dendrite/sync_api/_api/response/selector_cache_response.py b/dendrite/browser/sync_api/_api/response/selector_cache_response.py similarity index 100% rename from dendrite/sync_api/_api/response/selector_cache_response.py rename to dendrite/browser/sync_api/_api/response/selector_cache_response.py diff --git a/dendrite/sync_api/_api/response/session_response.py b/dendrite/browser/sync_api/_api/response/session_response.py similarity index 100% rename from dendrite/sync_api/_api/response/session_response.py rename to dendrite/browser/sync_api/_api/response/session_response.py diff --git a/dendrite/sync_api/_common/__init__.py b/dendrite/browser/sync_api/_common/__init__.py similarity index 100% rename from dendrite/sync_api/_common/__init__.py rename to dendrite/browser/sync_api/_common/__init__.py diff --git a/dendrite/sync_api/_common/constants.py b/dendrite/browser/sync_api/_common/constants.py similarity index 100% rename from dendrite/sync_api/_common/constants.py rename to dendrite/browser/sync_api/_common/constants.py diff --git a/dendrite/sync_api/_common/event_sync.py b/dendrite/browser/sync_api/_common/event_sync.py similarity index 100% rename from dendrite/sync_api/_common/event_sync.py rename to dendrite/browser/sync_api/_common/event_sync.py diff --git a/dendrite/sync_api/_common/status.py b/dendrite/browser/sync_api/_common/status.py similarity index 100% rename from dendrite/sync_api/_common/status.py rename to dendrite/browser/sync_api/_common/status.py diff --git a/dendrite/sync_api/_core/__init__.py b/dendrite/browser/sync_api/_core/__init__.py similarity index 100% rename from dendrite/sync_api/_core/__init__.py rename to dendrite/browser/sync_api/_core/__init__.py diff --git a/dendrite/sync_api/_core/_impl_browser.py b/dendrite/browser/sync_api/_core/_impl_browser.py similarity index 93% rename from dendrite/sync_api/_core/_impl_browser.py rename to dendrite/browser/sync_api/_core/_impl_browser.py index bbff259..67899cd 100644 --- a/dendrite/sync_api/_core/_impl_browser.py +++ b/dendrite/browser/sync_api/_core/_impl_browser.py @@ -2,8 +2,8 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_browser import Dendrite -from dendrite.sync_api._core._type_spec import PlaywrightPage + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage from playwright.sync_api import Download, Browser, Playwright diff --git a/dendrite/sync_api/_core/_impl_mapping.py b/dendrite/browser/sync_api/_core/_impl_mapping.py similarity index 63% rename from dendrite/sync_api/_core/_impl_mapping.py rename to dendrite/browser/sync_api/_core/_impl_mapping.py index fc0688f..8bc4e62 100644 --- a/dendrite/sync_api/_core/_impl_mapping.py +++ b/dendrite/browser/sync_api/_core/_impl_mapping.py @@ -1,10 +1,10 @@ from typing import Any, Dict, Optional, Type -from dendrite.sync_api._core._impl_browser import ImplBrowser, LocalImpl -from dendrite.sync_api._ext_impl.browserbase._impl import BrowserBaseImpl -from dendrite.sync_api._ext_impl.browserless._impl import BrowserlessImpl -from dendrite.remote.browserless_config import BrowserlessConfig -from dendrite.remote.browserbase_config import BrowserbaseConfig -from dendrite.remote import Providers +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser, LocalImpl +from dendrite.browser.sync_api._ext_impl.browserbase._impl import BrowserBaseImpl +from dendrite.browser.sync_api._ext_impl.browserless._impl import BrowserlessImpl +from dendrite.browser.remote.browserless_config import BrowserlessConfig +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser.remote import Providers IMPL_MAPPING: Dict[Type[Providers], Type[ImplBrowser]] = { BrowserbaseConfig: BrowserBaseImpl, diff --git a/dendrite/sync_api/_core/_js/__init__.py b/dendrite/browser/sync_api/_core/_js/__init__.py similarity index 100% rename from dendrite/sync_api/_core/_js/__init__.py rename to dendrite/browser/sync_api/_core/_js/__init__.py diff --git a/dendrite/sync_api/_core/_js/eventListenerPatch.js b/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js similarity index 100% rename from dendrite/sync_api/_core/_js/eventListenerPatch.js rename to dendrite/browser/sync_api/_core/_js/eventListenerPatch.js diff --git a/dendrite/async_api/_core/_js/generateDendriteIDs.js b/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js similarity index 100% rename from dendrite/async_api/_core/_js/generateDendriteIDs.js rename to dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js diff --git a/dendrite/async_api/_core/_js/generateDendriteIDsIframe.js b/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js similarity index 100% rename from dendrite/async_api/_core/_js/generateDendriteIDsIframe.js rename to dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js diff --git a/dendrite/sync_api/_core/_managers/__init__.py b/dendrite/browser/sync_api/_core/_managers/__init__.py similarity index 100% rename from dendrite/sync_api/_core/_managers/__init__.py rename to dendrite/browser/sync_api/_core/_managers/__init__.py diff --git a/dendrite/sync_api/_core/_managers/navigation_tracker.py b/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py similarity index 97% rename from dendrite/sync_api/_core/_managers/navigation_tracker.py rename to dendrite/browser/sync_api/_core/_managers/navigation_tracker.py index 8735d05..8dc632b 100644 --- a/dendrite/sync_api/_core/_managers/navigation_tracker.py +++ b/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_page import Page + from dendrite.browser.sync_api._core.dendrite_page import Page class NavigationTracker: diff --git a/dendrite/sync_api/_core/_managers/page_manager.py b/dendrite/browser/sync_api/_core/_managers/page_manager.py similarity index 93% rename from dendrite/sync_api/_core/_managers/page_manager.py rename to dendrite/browser/sync_api/_core/_managers/page_manager.py index b8e77d8..79734db 100644 --- a/dendrite/sync_api/_core/_managers/page_manager.py +++ b/dendrite/browser/sync_api/_core/_managers/page_manager.py @@ -3,9 +3,9 @@ from playwright.sync_api import BrowserContext, Download, FileChooser if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_browser import Dendrite -from dendrite.sync_api._core._type_spec import PlaywrightPage -from dendrite.sync_api._core.dendrite_page import Page + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core.dendrite_page import Page class PageManager: diff --git a/dendrite/sync_api/_core/_managers/screenshot_manager.py b/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py similarity index 96% rename from dendrite/sync_api/_core/_managers/screenshot_manager.py rename to dendrite/browser/sync_api/_core/_managers/screenshot_manager.py index a6f36b1..9a01f7c 100644 --- a/dendrite/sync_api/_core/_managers/screenshot_manager.py +++ b/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py @@ -1,7 +1,7 @@ import base64 import os from uuid import uuid4 -from dendrite.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage class ScreenshotManager: diff --git a/dendrite/sync_api/_core/_type_spec.py b/dendrite/browser/sync_api/_core/_type_spec.py similarity index 100% rename from dendrite/sync_api/_core/_type_spec.py rename to dendrite/browser/sync_api/_core/_type_spec.py diff --git a/dendrite/sync_api/_core/_utils.py b/dendrite/browser/sync_api/_core/_utils.py similarity index 86% rename from dendrite/sync_api/_core/_utils.py rename to dendrite/browser/sync_api/_core/_utils.py index 056b358..7993862 100644 --- a/dendrite/sync_api/_core/_utils.py +++ b/dendrite/browser/sync_api/_core/_utils.py @@ -2,15 +2,15 @@ from playwright.sync_api import FrameLocator, ElementHandle, Error, Frame from bs4 import BeautifulSoup from loguru import logger -from dendrite.sync_api._api.response.get_element_response import GetElementResponse -from dendrite.sync_api._core._type_spec import PlaywrightPage -from dendrite.sync_api._core.dendrite_element import Element -from dendrite.sync_api._core.models.response import ElementsResponse +from dendrite.browser.sync_api._api.response.get_element_response import GetElementResponse +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core.dendrite_element import Element +from dendrite.browser.sync_api._core.models.response import ElementsResponse if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_page import Page -from dendrite.sync_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT -from dendrite.sync_api._dom.util.mild_strip import mild_strip_in_place + from dendrite.browser.sync_api._core.dendrite_page import Page +from dendrite.browser.sync_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT +from dendrite.browser.sync_api._dom.util.mild_strip import mild_strip_in_place def expand_iframes(page: PlaywrightPage, page_soup: BeautifulSoup): diff --git a/dendrite/sync_api/_core/dendrite_browser.py b/dendrite/browser/sync_api/_core/dendrite_browser.py similarity index 90% rename from dendrite/sync_api/_core/dendrite_browser.py rename to dendrite/browser/sync_api/_core/dendrite_browser.py index 259841e..06017c5 100644 --- a/dendrite/sync_api/_core/dendrite_browser.py +++ b/dendrite/browser/sync_api/_core/dendrite_browser.py @@ -14,33 +14,33 @@ Error, FilePayload, ) -from dendrite.sync_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.sync_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.sync_api._common.event_sync import EventSync -from dendrite.sync_api._core._impl_browser import ImplBrowser -from dendrite.sync_api._core._impl_mapping import get_impl -from dendrite.sync_api._core._managers.page_manager import PageManager -from dendrite.sync_api._core._type_spec import PlaywrightPage -from dendrite.sync_api._core.dendrite_page import Page -from dendrite.sync_api._common.constants import STEALTH_ARGS -from dendrite.sync_api._core.mixin.ask import AskMixin -from dendrite.sync_api._core.mixin.click import ClickMixin -from dendrite.sync_api._core.mixin.extract import ExtractionMixin -from dendrite.sync_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.sync_api._core.mixin.keyboard import KeyboardMixin -from dendrite.sync_api._core.mixin.screenshot import ScreenshotMixin -from dendrite.sync_api._core.mixin.wait_for import WaitForMixin -from dendrite.sync_api._core.mixin.markdown import MarkdownMixin -from dendrite.sync_api._core.models.authentication import AuthSession -from dendrite.sync_api._core.models.api_config import APIConfig -from dendrite.sync_api._api.browser_api_client import BrowserAPIClient -from dendrite._common._exceptions.dendrite_exception import ( +from dendrite.browser.sync_api._api.dto.authenticate_dto import AuthenticateDTO +from dendrite.browser.sync_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO +from dendrite.browser.sync_api._common.event_sync import EventSync +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser +from dendrite.browser.sync_api._core._impl_mapping import get_impl +from dendrite.browser.sync_api._core._managers.page_manager import PageManager +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core.dendrite_page import Page +from dendrite.browser.sync_api._common.constants import STEALTH_ARGS +from dendrite.browser.sync_api._core.mixin.ask import AskMixin +from dendrite.browser.sync_api._core.mixin.click import ClickMixin +from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin +from dendrite.browser.sync_api._core.mixin.fill_fields import FillFieldsMixin +from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.sync_api._core.mixin.keyboard import KeyboardMixin +from dendrite.browser.sync_api._core.mixin.screenshot import ScreenshotMixin +from dendrite.browser.sync_api._core.mixin.wait_for import WaitForMixin +from dendrite.browser.sync_api._core.mixin.markdown import MarkdownMixin +from dendrite.browser.sync_api._core.models.authentication import AuthSession +from dendrite.browser.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._api.browser_api_client import BrowserAPIClient +from dendrite.browser._common._exceptions.dendrite_exception import ( BrowserNotLaunchedError, DendriteException, IncorrectOutcomeError, ) -from dendrite.remote import Providers +from dendrite.browser.remote import Providers class Dendrite( diff --git a/dendrite/sync_api/_core/dendrite_element.py b/dendrite/browser/sync_api/_core/dendrite_element.py similarity index 92% rename from dendrite/sync_api/_core/dendrite_element.py rename to dendrite/browser/sync_api/_core/dendrite_element.py index d73e788..7242768 100644 --- a/dendrite/sync_api/_core/dendrite_element.py +++ b/dendrite/browser/sync_api/_core/dendrite_element.py @@ -6,16 +6,16 @@ from typing import TYPE_CHECKING, Optional from loguru import logger from playwright.sync_api import Locator -from dendrite.sync_api._api.browser_api_client import BrowserAPIClient -from dendrite._common._exceptions.dendrite_exception import IncorrectOutcomeError +from dendrite.browser.sync_api._api.browser_api_client import BrowserAPIClient +from dendrite.browser._common._exceptions.dendrite_exception import IncorrectOutcomeError if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_browser import Dendrite -from dendrite.sync_api._core._managers.navigation_tracker import NavigationTracker -from dendrite.sync_api._core.models.page_diff_information import PageDiffInformation -from dendrite.sync_api._core._type_spec import Interaction -from dendrite.sync_api._api.response.interaction_response import InteractionResponse -from dendrite.sync_api._api.dto.make_interaction_dto import MakeInteractionDTO + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser.sync_api._core._managers.navigation_tracker import NavigationTracker +from dendrite.browser.sync_api._core.models.page_diff_information import PageDiffInformation +from dendrite.browser.sync_api._core._type_spec import Interaction +from dendrite.browser.sync_api._api.response.interaction_response import InteractionResponse +from dendrite.browser.sync_api._api.dto.make_interaction_dto import MakeInteractionDTO def perform_action(interaction_type: Interaction): diff --git a/dendrite/sync_api/_core/dendrite_page.py b/dendrite/browser/sync_api/_core/dendrite_page.py similarity index 90% rename from dendrite/sync_api/_core/dendrite_page.py rename to dendrite/browser/sync_api/_core/dendrite_page.py index b9cd048..947021b 100644 --- a/dendrite/sync_api/_core/dendrite_page.py +++ b/dendrite/browser/sync_api/_core/dendrite_page.py @@ -6,25 +6,25 @@ from bs4 import BeautifulSoup, Tag from loguru import logger from playwright.sync_api import FrameLocator, Keyboard, Download, FilePayload -from dendrite.sync_api._api.browser_api_client import BrowserAPIClient -from dendrite.sync_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT -from dendrite.sync_api._core._type_spec import PlaywrightPage -from dendrite.sync_api._core.dendrite_element import Element -from dendrite.sync_api._core.mixin.ask import AskMixin -from dendrite.sync_api._core.mixin.click import ClickMixin -from dendrite.sync_api._core.mixin.extract import ExtractionMixin -from dendrite.sync_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.sync_api._core.mixin.keyboard import KeyboardMixin -from dendrite.sync_api._core.mixin.markdown import MarkdownMixin -from dendrite.sync_api._core.mixin.wait_for import WaitForMixin -from dendrite.sync_api._core.models.page_information import PageInformation +from dendrite.browser.sync_api._api.browser_api_client import BrowserAPIClient +from dendrite.browser.sync_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core.dendrite_element import Element +from dendrite.browser.sync_api._core.mixin.ask import AskMixin +from dendrite.browser.sync_api._core.mixin.click import ClickMixin +from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin +from dendrite.browser.sync_api._core.mixin.fill_fields import FillFieldsMixin +from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.sync_api._core.mixin.keyboard import KeyboardMixin +from dendrite.browser.sync_api._core.mixin.markdown import MarkdownMixin +from dendrite.browser.sync_api._core.mixin.wait_for import WaitForMixin +from dendrite.browser.sync_api._core.models.page_information import PageInformation if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_browser import Dendrite -from dendrite.sync_api._core._managers.screenshot_manager import ScreenshotManager -from dendrite._common._exceptions.dendrite_exception import DendriteException -from dendrite.sync_api._core._utils import expand_iframes + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser.sync_api._core._managers.screenshot_manager import ScreenshotManager +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core._utils import expand_iframes class Page( diff --git a/dendrite/sync_api/_core/mixin/ask.py b/dendrite/browser/sync_api/_core/mixin/ask.py similarity index 96% rename from dendrite/sync_api/_core/mixin/ask.py rename to dendrite/browser/sync_api/_core/mixin/ask.py index ca028f8..ff1da2f 100644 --- a/dendrite/sync_api/_core/mixin/ask.py +++ b/dendrite/browser/sync_api/_core/mixin/ask.py @@ -2,16 +2,16 @@ import time from typing import Optional, Type, overload from loguru import logger -from dendrite.sync_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.sync_api._core._type_spec import ( +from dendrite.browser.sync_api._api.dto.ask_page_dto import AskPageDTO +from dendrite.browser.sync_api._core._type_spec import ( JsonSchema, PydanticModel, TypeSpec, convert_to_type_spec, to_json_schema, ) -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException TIMEOUT_INTERVAL = [150, 450, 1000] diff --git a/dendrite/sync_api/_core/mixin/click.py b/dendrite/browser/sync_api/_core/mixin/click.py similarity index 85% rename from dendrite/sync_api/_core/mixin/click.py rename to dendrite/browser/sync_api/_core/mixin/click.py index 097eccb..80fc071 100644 --- a/dendrite/sync_api/_core/mixin/click.py +++ b/dendrite/browser/sync_api/_core/mixin/click.py @@ -1,9 +1,9 @@ import time from typing import Any, Optional -from dendrite.sync_api._api.response.interaction_response import InteractionResponse -from dendrite.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._api.response.interaction_response import InteractionResponse +from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException class ClickMixin(GetElementMixin, DendritePageProtocol): diff --git a/dendrite/sync_api/_core/mixin/extract.py b/dendrite/browser/sync_api/_core/mixin/extract.py similarity index 93% rename from dendrite/sync_api/_core/mixin/extract.py rename to dendrite/browser/sync_api/_core/mixin/extract.py index 15ca694..108806f 100644 --- a/dendrite/sync_api/_core/mixin/extract.py +++ b/dendrite/browser/sync_api/_core/mixin/extract.py @@ -1,18 +1,18 @@ import time import time from typing import Any, Optional, Type, overload, List -from dendrite.sync_api._api.dto.extract_dto import ExtractDTO -from dendrite.sync_api._api.response.cache_extract_response import CacheExtractResponse -from dendrite.sync_api._api.response.extract_response import ExtractResponse -from dendrite.sync_api._core._type_spec import ( +from dendrite.browser.sync_api._api.dto.extract_dto import ExtractDTO +from dendrite.browser.sync_api._api.response.cache_extract_response import CacheExtractResponse +from dendrite.browser.sync_api._api.response.extract_response import ExtractResponse +from dendrite.browser.sync_api._core._type_spec import ( JsonSchema, PydanticModel, TypeSpec, convert_to_type_spec, to_json_schema, ) -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.sync_api._core._managers.navigation_tracker import NavigationTracker +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.sync_api._core._managers.navigation_tracker import NavigationTracker from loguru import logger CACHE_TIMEOUT = 5 diff --git a/dendrite/sync_api/_core/mixin/fill_fields.py b/dendrite/browser/sync_api/_core/mixin/fill_fields.py similarity index 89% rename from dendrite/sync_api/_core/mixin/fill_fields.py rename to dendrite/browser/sync_api/_core/mixin/fill_fields.py index 792ab24..885a54c 100644 --- a/dendrite/sync_api/_core/mixin/fill_fields.py +++ b/dendrite/browser/sync_api/_core/mixin/fill_fields.py @@ -1,9 +1,9 @@ import time from typing import Any, Dict, Optional -from dendrite.sync_api._api.response.interaction_response import InteractionResponse -from dendrite.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._api.response.interaction_response import InteractionResponse +from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException class FillFieldsMixin(GetElementMixin, DendritePageProtocol): diff --git a/dendrite/sync_api/_core/mixin/get_element.py b/dendrite/browser/sync_api/_core/mixin/get_element.py similarity index 94% rename from dendrite/sync_api/_core/mixin/get_element.py rename to dendrite/browser/sync_api/_core/mixin/get_element.py index ded124e..3fe7511 100644 --- a/dendrite/sync_api/_core/mixin/get_element.py +++ b/dendrite/browser/sync_api/_core/mixin/get_element.py @@ -2,14 +2,14 @@ import time from typing import Dict, List, Literal, Optional, Union, overload from loguru import logger -from dendrite.sync_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.sync_api._api.response.get_element_response import GetElementResponse -from dendrite.sync_api._api.dto.get_elements_dto import CheckSelectorCacheDTO -from dendrite.sync_api._core._utils import get_elements_from_selectors_soup -from dendrite.sync_api._core.dendrite_element import Element -from dendrite.sync_api._core.models.response import ElementsResponse -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.sync_api._core.models.api_config import APIConfig +from dendrite.browser.sync_api._api.dto.get_elements_dto import GetElementsDTO +from dendrite.browser.sync_api._api.response.get_element_response import GetElementResponse +from dendrite.browser.sync_api._api.dto.get_elements_dto import CheckSelectorCacheDTO +from dendrite.browser.sync_api._core._utils import get_elements_from_selectors_soup +from dendrite.browser.sync_api._core.dendrite_element import Element +from dendrite.browser.sync_api._core.models.response import ElementsResponse +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.sync_api._core.models.api_config import APIConfig CACHE_TIMEOUT = 5 @@ -112,6 +112,7 @@ def get_element( Returns: Element: The retrieved element. """ + logger.info(f"Getting element for prompt: {prompt}") return self._get_element( prompt, only_one=True, use_cache=use_cache, timeout=timeout / 1000 ) diff --git a/dendrite/sync_api/_core/mixin/keyboard.py b/dendrite/browser/sync_api/_core/mixin/keyboard.py similarity index 91% rename from dendrite/sync_api/_core/mixin/keyboard.py rename to dendrite/browser/sync_api/_core/mixin/keyboard.py index e3ed73a..b728770 100644 --- a/dendrite/sync_api/_core/mixin/keyboard.py +++ b/dendrite/browser/sync_api/_core/mixin/keyboard.py @@ -1,6 +1,6 @@ from typing import Any, Union, Literal -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException class KeyboardMixin(DendritePageProtocol): diff --git a/dendrite/sync_api/_core/mixin/markdown.py b/dendrite/browser/sync_api/_core/mixin/markdown.py similarity index 87% rename from dendrite/sync_api/_core/mixin/markdown.py rename to dendrite/browser/sync_api/_core/mixin/markdown.py index f094330..3125cff 100644 --- a/dendrite/sync_api/_core/mixin/markdown.py +++ b/dendrite/browser/sync_api/_core/mixin/markdown.py @@ -1,8 +1,8 @@ from typing import Optional from bs4 import BeautifulSoup import re -from dendrite.sync_api._core.mixin.extract import ExtractionMixin -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol from markdownify import markdownify as md diff --git a/dendrite/sync_api/_core/mixin/screenshot.py b/dendrite/browser/sync_api/_core/mixin/screenshot.py similarity index 87% rename from dendrite/sync_api/_core/mixin/screenshot.py rename to dendrite/browser/sync_api/_core/mixin/screenshot.py index 3495b4c..bd3fab2 100644 --- a/dendrite/sync_api/_core/mixin/screenshot.py +++ b/dendrite/browser/sync_api/_core/mixin/screenshot.py @@ -1,4 +1,4 @@ -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol class ScreenshotMixin(DendritePageProtocol): diff --git a/dendrite/sync_api/_core/mixin/wait_for.py b/dendrite/browser/sync_api/_core/mixin/wait_for.py similarity index 86% rename from dendrite/sync_api/_core/mixin/wait_for.py rename to dendrite/browser/sync_api/_core/mixin/wait_for.py index 76cac15..56b5bfc 100644 --- a/dendrite/sync_api/_core/mixin/wait_for.py +++ b/dendrite/browser/sync_api/_core/mixin/wait_for.py @@ -1,9 +1,9 @@ import time import time -from dendrite.sync_api._core.mixin.ask import AskMixin -from dendrite.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite._common._exceptions.dendrite_exception import PageConditionNotMet -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core.mixin.ask import AskMixin +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser._common._exceptions.dendrite_exception import PageConditionNotMet +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException from loguru import logger diff --git a/dendrite/sync_api/_core/models/__init__.py b/dendrite/browser/sync_api/_core/models/__init__.py similarity index 100% rename from dendrite/sync_api/_core/models/__init__.py rename to dendrite/browser/sync_api/_core/models/__init__.py diff --git a/dendrite/sync_api/_core/models/api_config.py b/dendrite/browser/sync_api/_core/models/api_config.py similarity index 93% rename from dendrite/sync_api/_core/models/api_config.py rename to dendrite/browser/sync_api/_core/models/api_config.py index 7d90502..756aa5b 100644 --- a/dendrite/sync_api/_core/models/api_config.py +++ b/dendrite/browser/sync_api/_core/models/api_config.py @@ -1,6 +1,6 @@ from typing import Optional from pydantic import BaseModel, model_validator -from dendrite._common._exceptions.dendrite_exception import MissingApiKeyError +from dendrite.browser._common._exceptions.dendrite_exception import MissingApiKeyError class APIConfig(BaseModel): diff --git a/dendrite/sync_api/_core/models/authentication.py b/dendrite/browser/sync_api/_core/models/authentication.py similarity index 100% rename from dendrite/sync_api/_core/models/authentication.py rename to dendrite/browser/sync_api/_core/models/authentication.py diff --git a/dendrite/sync_api/_core/models/download_interface.py b/dendrite/browser/sync_api/_core/models/download_interface.py similarity index 100% rename from dendrite/sync_api/_core/models/download_interface.py rename to dendrite/browser/sync_api/_core/models/download_interface.py diff --git a/dendrite/sync_api/_core/models/page_diff_information.py b/dendrite/browser/sync_api/_core/models/page_diff_information.py similarity index 61% rename from dendrite/sync_api/_core/models/page_diff_information.py rename to dendrite/browser/sync_api/_core/models/page_diff_information.py index d41d1fe..0dadf97 100644 --- a/dendrite/sync_api/_core/models/page_diff_information.py +++ b/dendrite/browser/sync_api/_core/models/page_diff_information.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from dendrite.sync_api._core.models.page_information import PageInformation +from dendrite.browser.sync_api._core.models.page_information import PageInformation class PageDiffInformation(BaseModel): diff --git a/dendrite/sync_api/_core/models/page_information.py b/dendrite/browser/sync_api/_core/models/page_information.py similarity index 100% rename from dendrite/sync_api/_core/models/page_information.py rename to dendrite/browser/sync_api/_core/models/page_information.py diff --git a/dendrite/sync_api/_core/models/response.py b/dendrite/browser/sync_api/_core/models/response.py similarity index 96% rename from dendrite/sync_api/_core/models/response.py rename to dendrite/browser/sync_api/_core/models/response.py index 76225c3..50d9a23 100644 --- a/dendrite/sync_api/_core/models/response.py +++ b/dendrite/browser/sync_api/_core/models/response.py @@ -1,5 +1,5 @@ from typing import Dict, Iterator -from dendrite.sync_api._core.dendrite_element import Element +from dendrite.browser.sync_api._core.dendrite_element import Element class ElementsResponse: diff --git a/dendrite/sync_api/_core/protocol/page_protocol.py b/dendrite/browser/sync_api/_core/protocol/page_protocol.py similarity index 63% rename from dendrite/sync_api/_core/protocol/page_protocol.py rename to dendrite/browser/sync_api/_core/protocol/page_protocol.py index 17b5e9b..13cb990 100644 --- a/dendrite/sync_api/_core/protocol/page_protocol.py +++ b/dendrite/browser/sync_api/_core/protocol/page_protocol.py @@ -1,9 +1,9 @@ from typing import TYPE_CHECKING, Protocol -from dendrite.sync_api._api.browser_api_client import BrowserAPIClient +from dendrite.browser.sync_api._api.browser_api_client import BrowserAPIClient if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_page import Page - from dendrite.sync_api._core.dendrite_browser import Dendrite + from dendrite.browser.sync_api._core.dendrite_page import Page + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite class DendritePageProtocol(Protocol): diff --git a/dendrite/sync_api/_dom/__init__.py b/dendrite/browser/sync_api/_dom/__init__.py similarity index 100% rename from dendrite/sync_api/_dom/__init__.py rename to dendrite/browser/sync_api/_dom/__init__.py diff --git a/dendrite/sync_api/_dom/util/mild_strip.py b/dendrite/browser/sync_api/_dom/util/mild_strip.py similarity index 100% rename from dendrite/sync_api/_dom/util/mild_strip.py rename to dendrite/browser/sync_api/_dom/util/mild_strip.py diff --git a/dendrite/sync_api/_ext_impl/__init__.py b/dendrite/browser/sync_api/_ext_impl/__init__.py similarity index 100% rename from dendrite/sync_api/_ext_impl/__init__.py rename to dendrite/browser/sync_api/_ext_impl/__init__.py diff --git a/dendrite/sync_api/_ext_impl/browserbase/__init__.py b/dendrite/browser/sync_api/_ext_impl/browserbase/__init__.py similarity index 100% rename from dendrite/sync_api/_ext_impl/browserbase/__init__.py rename to dendrite/browser/sync_api/_ext_impl/browserbase/__init__.py diff --git a/dendrite/sync_api/_ext_impl/browserbase/_client.py b/dendrite/browser/sync_api/_ext_impl/browserbase/_client.py similarity index 96% rename from dendrite/sync_api/_ext_impl/browserbase/_client.py rename to dendrite/browser/sync_api/_ext_impl/browserbase/_client.py index 5d862e2..81d2f0f 100644 --- a/dendrite/sync_api/_ext_impl/browserbase/_client.py +++ b/dendrite/browser/sync_api/_ext_impl/browserbase/_client.py @@ -4,7 +4,7 @@ from typing import Optional, Union import httpx from loguru import logger -from dendrite._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException class BrowserbaseClient: diff --git a/dendrite/sync_api/_ext_impl/browserbase/_download.py b/dendrite/browser/sync_api/_ext_impl/browserbase/_download.py similarity index 91% rename from dendrite/sync_api/_ext_impl/browserbase/_download.py rename to dendrite/browser/sync_api/_ext_impl/browserbase/_download.py index e669ba1..14756f9 100644 --- a/dendrite/sync_api/_ext_impl/browserbase/_download.py +++ b/dendrite/browser/sync_api/_ext_impl/browserbase/_download.py @@ -5,8 +5,8 @@ import zipfile from loguru import logger from playwright.sync_api import Download -from dendrite.sync_api._core.models.download_interface import DownloadInterface -from dendrite.sync_api._ext_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.sync_api._core.models.download_interface import DownloadInterface +from dendrite.browser.sync_api._ext_impl.browserbase._client import BrowserbaseClient class BrowserbaseDownload(DownloadInterface): diff --git a/dendrite/sync_api/_ext_impl/browserbase/_impl.py b/dendrite/browser/sync_api/_ext_impl/browserbase/_impl.py similarity index 78% rename from dendrite/sync_api/_ext_impl/browserbase/_impl.py rename to dendrite/browser/sync_api/_ext_impl/browserbase/_impl.py index 453c6b6..138a6e4 100644 --- a/dendrite/sync_api/_ext_impl/browserbase/_impl.py +++ b/dendrite/browser/sync_api/_ext_impl/browserbase/_impl.py @@ -1,15 +1,15 @@ from typing import TYPE_CHECKING, Optional -from dendrite._common._exceptions.dendrite_exception import BrowserNotLaunchedError -from dendrite.sync_api._core._impl_browser import ImplBrowser -from dendrite.sync_api._core._type_spec import PlaywrightPage -from dendrite.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser._common._exceptions.dendrite_exception import BrowserNotLaunchedError +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_browser import Dendrite -from dendrite.sync_api._ext_impl.browserbase._client import BrowserbaseClient + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser.sync_api._ext_impl.browserbase._client import BrowserbaseClient from playwright.sync_api import Playwright from loguru import logger -from dendrite.sync_api._ext_impl.browserbase._download import BrowserbaseDownload +from dendrite.browser.sync_api._ext_impl.browserbase._download import BrowserbaseDownload class BrowserBaseImpl(ImplBrowser): diff --git a/dendrite/sync_api/_ext_impl/browserless/__init__.py b/dendrite/browser/sync_api/_ext_impl/browserless/__init__.py similarity index 100% rename from dendrite/sync_api/_ext_impl/browserless/__init__.py rename to dendrite/browser/sync_api/_ext_impl/browserless/__init__.py diff --git a/dendrite/sync_api/_ext_impl/browserless/_impl.py b/dendrite/browser/sync_api/_ext_impl/browserless/_impl.py similarity index 72% rename from dendrite/sync_api/_ext_impl/browserless/_impl.py rename to dendrite/browser/sync_api/_ext_impl/browserless/_impl.py index 5d888e6..2e2646f 100644 --- a/dendrite/sync_api/_ext_impl/browserless/_impl.py +++ b/dendrite/browser/sync_api/_ext_impl/browserless/_impl.py @@ -1,17 +1,17 @@ import json from typing import TYPE_CHECKING, Optional -from dendrite._common._exceptions.dendrite_exception import BrowserNotLaunchedError -from dendrite.sync_api._core._impl_browser import ImplBrowser -from dendrite.sync_api._core._type_spec import PlaywrightPage -from dendrite.remote.browserless_config import BrowserlessConfig +from dendrite.browser._common._exceptions.dendrite_exception import BrowserNotLaunchedError +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.remote.browserless_config import BrowserlessConfig if TYPE_CHECKING: - from dendrite.sync_api._core.dendrite_browser import Dendrite -from dendrite.sync_api._ext_impl.browserbase._client import BrowserbaseClient + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser.sync_api._ext_impl.browserbase._client import BrowserbaseClient from playwright.sync_api import Playwright from loguru import logger import urllib.parse -from dendrite.sync_api._ext_impl.browserbase._download import BrowserbaseDownload +from dendrite.browser.sync_api._ext_impl.browserbase._download import BrowserbaseDownload class BrowserlessImpl(ImplBrowser): diff --git a/dendrite/exceptions/__init__.py b/dendrite/exceptions/__init__.py index fa5ff25..7aff356 100644 --- a/dendrite/exceptions/__init__.py +++ b/dendrite/exceptions/__init__.py @@ -1,4 +1,4 @@ -from .._common._exceptions.dendrite_exception import ( +from ..browser._common._exceptions.dendrite_exception import ( BaseDendriteException, DendriteException, IncorrectOutcomeError, diff --git a/dendrite/logic/hosted/_api/__init__.py b/dendrite/logic/hosted/_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/hosted/_api/_http_client.py b/dendrite/logic/hosted/_api/_http_client.py new file mode 100644 index 0000000..72777e2 --- /dev/null +++ b/dendrite/logic/hosted/_api/_http_client.py @@ -0,0 +1,65 @@ +import os +from typing import Optional +import httpx +from loguru import logger + +from dendrite.browser.async_api._core.models.api_config import APIConfig + + +class HTTPClient: + def __init__(self, api_config: APIConfig, session_id: Optional[str] = None): + self.api_key = api_config.dendrite_api_key + self.session_id = session_id + self.base_url = self.resolve_base_url() + + def resolve_base_url(self): + base_url = ( + "http://localhost:8000/api/v1" + if os.environ.get("DENDRITE_DEV") + else "https://dendrite-server.azurewebsites.net/api/v1" + ) + return base_url + + async def send_request( + self, + endpoint: str, + params: Optional[dict] = None, + data: Optional[dict] = None, + headers: Optional[dict] = None, + method: str = "GET", + ) -> httpx.Response: + url = f"{self.base_url}/{endpoint}" + + headers = headers or {} + headers["Content-Type"] = "application/json" + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + if self.session_id: + headers["X-Session-ID"] = self.session_id + + async with httpx.AsyncClient(timeout=300) as client: + try: + response = await client.request( + method, url, params=params, json=data, headers=headers + ) + response.raise_for_status() + # logger.debug( + # f"{method} to '{url}', that took: { time.time() - start_time }\n\nResponse: {dict_res}\n\n" + # ) + return response + except httpx.HTTPStatusError as http_err: + logger.debug( + f"HTTP error occurred: {http_err.response.status_code}: {http_err.response.text}" + ) + raise + except httpx.ConnectError as connect_err: + logger.error( + f"Connection error occurred: {connect_err}. {url} Server might be down" + ) + raise + except httpx.RequestError as req_err: + # logger.debug(f"Request error occurred: {req_err}") + raise + except Exception as err: + # logger.debug(f"An error occurred: {err}") + raise diff --git a/dendrite/logic/hosted/_api/browser_api_client.py b/dendrite/logic/hosted/_api/browser_api_client.py new file mode 100644 index 0000000..b52738f --- /dev/null +++ b/dendrite/logic/hosted/_api/browser_api_client.py @@ -0,0 +1,120 @@ +from typing import Optional + +from loguru import logger +from dendrite.browser.async_api._api.response.cache_extract_response import ( + CacheExtractResponse, +) +from dendrite.browser.async_api._api.response.selector_cache_response import ( + SelectorCacheResponse, +) +from dendrite.browser.async_api._core.models.authentication import AuthSession +from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse +from dendrite.browser.async_api._api.dto.ask_page_dto import AskPageDTO +from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO +from dendrite.browser.async_api._api.dto.get_elements_dto import GetElementsDTO +from dendrite.browser.async_api._api.dto.make_interaction_dto import MakeInteractionDTO +from dendrite.browser.async_api._api.dto.extract_dto import ExtractDTO +from dendrite.browser.async_api._api.dto.try_run_script_dto import TryRunScriptDTO +from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO +from dendrite.browser.async_api._api.response.ask_page_response import AskPageResponse +from dendrite.browser.async_api._api.response.interaction_response import ( + InteractionResponse, +) +from dendrite.browser.async_api._api.response.extract_response import ExtractResponse +from dendrite.browser.async_api._api._http_client import HTTPClient +from dendrite.browser._common._exceptions.dendrite_exception import ( + InvalidAuthSessionError, +) +from dendrite.browser.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO + + +class BrowserAPIClient(HTTPClient): + + async def authenticate(self, dto: AuthenticateDTO): + res = await self.send_request( + "actions/authenticate", data=dto.model_dump(), method="POST" + ) + + if res.status_code == 204: + raise InvalidAuthSessionError(domain=dto.domains) + + return AuthSession(**res.json()) + + async def upload_auth_session(self, dto: UploadAuthSessionDTO): + await self.send_request( + "actions/upload-auth-session", data=dto.dict(), method="POST" + ) + + async def check_selector_cache( + self, dto: CheckSelectorCacheDTO + ) -> SelectorCacheResponse: + res = await self.send_request( + "actions/check-selector-cache", data=dto.dict(), method="POST" + ) + return SelectorCacheResponse(**res.json()) + + async def get_interactions_selector( + self, dto: GetElementsDTO + ) -> GetElementResponse: + res = await self.send_request( + "actions/get-interaction-selector", data=dto.dict(), method="POST" + ) + return GetElementResponse(**res.json()) + + async def make_interaction(self, dto: MakeInteractionDTO) -> InteractionResponse: + res = await self.send_request( + "actions/make-interaction", data=dto.dict(), method="POST" + ) + res_dict = res.json() + return InteractionResponse( + status=res_dict["status"], message=res_dict["message"] + ) + + async def check_extract_cache(self, dto: ExtractDTO) -> CacheExtractResponse: + res = await self.send_request( + "actions/check-extract-cache", data=dto.dict(), method="POST" + ) + return CacheExtractResponse(**res.json()) + + async def extract(self, dto: ExtractDTO) -> ExtractResponse: + res = await self.send_request( + "actions/extract-page", data=dto.dict(), method="POST" + ) + res_dict = res.json() + return ExtractResponse( + status=res_dict["status"], + message=res_dict["message"], + return_data=res_dict["return_data"], + created_script=res_dict.get("created_script", None), + used_cache=res_dict.get("used_cache", False), + ) + + async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: + res = await self.send_request( + "actions/ask-page", data=dto.dict(), method="POST" + ) + res_dict = res.json() + return AskPageResponse( + status=res_dict["status"], + description=res_dict["description"], + return_data=res_dict["return_data"], + ) + + async def try_run_cached(self, dto: TryRunScriptDTO) -> Optional[ExtractResponse]: + res = await self.send_request( + "actions/try-run-cached", data=dto.dict(), method="POST" + ) + if res is None: + return None + res_dict = res.json() + loaded_value = res_dict["return_data"] + if loaded_value is None: + return None + + return ExtractResponse( + status=res_dict["status"], + message=res_dict["message"], + return_data=loaded_value, + created_script=res_dict.get("created_script", None), + used_cache=res_dict.get("used_cache", False), + ) diff --git a/dendrite/logic/hosted/_api/dto/__init__.py b/dendrite/logic/hosted/_api/dto/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/hosted/_api/dto/ask_page_dto.py b/dendrite/logic/hosted/_api/dto/ask_page_dto.py new file mode 100644 index 0000000..f2dcaf3 --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/ask_page_dto.py @@ -0,0 +1,11 @@ +from typing import Any, Optional +from pydantic import BaseModel +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation + + +class AskPageDTO(BaseModel): + prompt: str + return_schema: Optional[Any] + page_information: PageInformation + api_config: APIConfig diff --git a/dendrite/logic/hosted/_api/dto/authenticate_dto.py b/dendrite/logic/hosted/_api/dto/authenticate_dto.py new file mode 100644 index 0000000..f5a1de7 --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/authenticate_dto.py @@ -0,0 +1,6 @@ +from typing import Union +from pydantic import BaseModel + + +class AuthenticateDTO(BaseModel): + domains: Union[str, list[str]] diff --git a/dendrite/logic/hosted/_api/dto/extract_dto.py b/dendrite/logic/hosted/_api/dto/extract_dto.py new file mode 100644 index 0000000..8cf1cc7 --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/extract_dto.py @@ -0,0 +1,25 @@ +import json +from typing import Any +from pydantic import BaseModel +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation + + +class ExtractDTO(BaseModel): + page_information: PageInformation + api_config: APIConfig + prompt: str + return_data_json_schema: Any + use_screenshot: bool = False + use_cache: bool = True + force_use_cache: bool = False + + @property + def combined_prompt(self) -> str: + + json_schema_prompt = ( + "" + if self.return_data_json_schema is None + else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" + ) + return f"Task: {self.prompt}{json_schema_prompt}" diff --git a/dendrite/logic/hosted/_api/dto/get_elements_dto.py b/dendrite/logic/hosted/_api/dto/get_elements_dto.py new file mode 100644 index 0000000..636c896 --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/get_elements_dto.py @@ -0,0 +1,19 @@ +from typing import Dict, Union +from pydantic import BaseModel + +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation + + +class CheckSelectorCacheDTO(BaseModel): + url: str + prompt: Union[str, Dict[str, str]] + + +class GetElementsDTO(BaseModel): + page_information: PageInformation + prompt: Union[str, Dict[str, str]] + api_config: APIConfig + use_cache: bool = True + only_one: bool + force_use_cache: bool = False diff --git a/dendrite/logic/hosted/_api/dto/get_interaction_dto.py b/dendrite/logic/hosted/_api/dto/get_interaction_dto.py new file mode 100644 index 0000000..93889c7 --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/get_interaction_dto.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel + +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation + + +class GetInteractionDTO(BaseModel): + page_information: PageInformation + api_config: APIConfig + prompt: str diff --git a/dendrite/logic/hosted/_api/dto/get_session_dto.py b/dendrite/logic/hosted/_api/dto/get_session_dto.py new file mode 100644 index 0000000..6414cc3 --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/get_session_dto.py @@ -0,0 +1,7 @@ +from typing import List +from pydantic import BaseModel + + +class GetSessionDTO(BaseModel): + user_id: str + domain: str diff --git a/dendrite/logic/hosted/_api/dto/google_search_dto.py b/dendrite/logic/hosted/_api/dto/google_search_dto.py new file mode 100644 index 0000000..6c55615 --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/google_search_dto.py @@ -0,0 +1,12 @@ +from typing import Optional +from pydantic import BaseModel +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation + + +class GoogleSearchDTO(BaseModel): + query: str + country: Optional[str] = None + filter_results_prompt: Optional[str] = None + page_information: PageInformation + api_config: APIConfig diff --git a/dendrite/logic/hosted/_api/dto/make_interaction_dto.py b/dendrite/logic/hosted/_api/dto/make_interaction_dto.py new file mode 100644 index 0000000..c0592ad --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/make_interaction_dto.py @@ -0,0 +1,19 @@ +from typing import Literal, Optional +from pydantic import BaseModel +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_diff_information import ( + PageDiffInformation, +) + + +InteractionType = Literal["click", "fill", "hover"] + + +class MakeInteractionDTO(BaseModel): + url: str + dendrite_id: str + interaction_type: InteractionType + value: Optional[str] = None + expected_outcome: Optional[str] + page_delta_information: PageDiffInformation + api_config: APIConfig diff --git a/dendrite/logic/hosted/_api/dto/try_run_script_dto.py b/dendrite/logic/hosted/_api/dto/try_run_script_dto.py new file mode 100644 index 0000000..e283806 --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/try_run_script_dto.py @@ -0,0 +1,14 @@ +from typing import Any, Optional +from pydantic import BaseModel +from dendrite.browser.async_api._core.models.api_config import APIConfig + + +class TryRunScriptDTO(BaseModel): + url: str + raw_html: str + api_config: APIConfig + prompt: str + db_prompt: Optional[str] = ( + None # If you wish to cache a script based of a fixed prompt use this value + ) + return_data_json_schema: Any diff --git a/dendrite/logic/hosted/_api/dto/upload_auth_session_dto.py b/dendrite/logic/hosted/_api/dto/upload_auth_session_dto.py new file mode 100644 index 0000000..1697fdf --- /dev/null +++ b/dendrite/logic/hosted/_api/dto/upload_auth_session_dto.py @@ -0,0 +1,11 @@ +from pydantic import BaseModel + +from dendrite.browser.async_api._core.models.authentication import ( + AuthSession, + StorageState, +) + + +class UploadAuthSessionDTO(BaseModel): + auth_data: AuthSession + storage_state: StorageState diff --git a/dendrite/logic/hosted/_api/response/__init__.py b/dendrite/logic/hosted/_api/response/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/hosted/_api/response/ask_page_response.py b/dendrite/logic/hosted/_api/response/ask_page_response.py new file mode 100644 index 0000000..4ec747a --- /dev/null +++ b/dendrite/logic/hosted/_api/response/ask_page_response.py @@ -0,0 +1,11 @@ +from typing import Generic, Literal, TypeVar +from pydantic import BaseModel + + +T = TypeVar("T") + + +class AskPageResponse(BaseModel, Generic[T]): + status: Literal["success", "error"] + return_data: T + description: str diff --git a/dendrite/logic/hosted/_api/response/cache_extract_response.py b/dendrite/logic/hosted/_api/response/cache_extract_response.py new file mode 100644 index 0000000..463d03b --- /dev/null +++ b/dendrite/logic/hosted/_api/response/cache_extract_response.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + + +class CacheExtractResponse(BaseModel): + exists: bool diff --git a/dendrite/logic/hosted/_api/response/extract_response.py b/dendrite/logic/hosted/_api/response/extract_response.py new file mode 100644 index 0000000..dd7f6d3 --- /dev/null +++ b/dendrite/logic/hosted/_api/response/extract_response.py @@ -0,0 +1,15 @@ +from typing import Generic, Optional, TypeVar +from pydantic import BaseModel + +from dendrite.browser.async_api._common.status import Status + + +T = TypeVar("T") + + +class ExtractResponse(BaseModel, Generic[T]): + return_data: T + message: str + created_script: Optional[str] = None + status: Status + used_cache: bool diff --git a/dendrite/logic/hosted/_api/response/get_element_response.py b/dendrite/logic/hosted/_api/response/get_element_response.py new file mode 100644 index 0000000..8fb8fa5 --- /dev/null +++ b/dendrite/logic/hosted/_api/response/get_element_response.py @@ -0,0 +1,12 @@ +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel + +from dendrite.browser.async_api._common.status import Status + + +class GetElementResponse(BaseModel): + status: Status + selectors: Optional[Union[List[str], Dict[str, List[str]]]] = None + message: str = "" + used_cache: bool = False diff --git a/dendrite/logic/hosted/_api/response/google_search_response.py b/dendrite/logic/hosted/_api/response/google_search_response.py new file mode 100644 index 0000000..d435b71 --- /dev/null +++ b/dendrite/logic/hosted/_api/response/google_search_response.py @@ -0,0 +1,12 @@ +from typing import List +from pydantic import BaseModel + + +class SearchResult(BaseModel): + url: str + title: str + description: str + + +class GoogleSearchResponse(BaseModel): + results: List[SearchResult] diff --git a/dendrite/logic/hosted/_api/response/interaction_response.py b/dendrite/logic/hosted/_api/response/interaction_response.py new file mode 100644 index 0000000..3dd2e49 --- /dev/null +++ b/dendrite/logic/hosted/_api/response/interaction_response.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel +from dendrite.browser.async_api._common.status import Status + + +class InteractionResponse(BaseModel): + message: str + status: Status diff --git a/dendrite/logic/hosted/_api/response/selector_cache_response.py b/dendrite/logic/hosted/_api/response/selector_cache_response.py new file mode 100644 index 0000000..4c0e388 --- /dev/null +++ b/dendrite/logic/hosted/_api/response/selector_cache_response.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + + +class SelectorCacheResponse(BaseModel): + exists: bool diff --git a/dendrite/logic/hosted/_api/response/session_response.py b/dendrite/logic/hosted/_api/response/session_response.py new file mode 100644 index 0000000..2d03b97 --- /dev/null +++ b/dendrite/logic/hosted/_api/response/session_response.py @@ -0,0 +1,7 @@ +from typing import List +from pydantic import BaseModel + + +class SessionResponse(BaseModel): + cookies: List[dict] + origins_storage: List[dict] diff --git a/dendrite/logic/hosted/async_api_impl.py b/dendrite/logic/hosted/async_api_impl.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/interfaces/async_api.py b/dendrite/logic/interfaces/async_api.py new file mode 100644 index 0000000..663e37f --- /dev/null +++ b/dendrite/logic/interfaces/async_api.py @@ -0,0 +1,48 @@ +# dendrite/browser/async_api/_api/protocols.py +from typing import Protocol, Optional +from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO +from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO +from dendrite.browser.async_api._api.dto.get_elements_dto import ( + GetElementsDTO, + CheckSelectorCacheDTO, +) +from dendrite.browser.async_api._api.dto.make_interaction_dto import MakeInteractionDTO +from dendrite.browser.async_api._api.dto.extract_dto import ExtractDTO +from dendrite.browser.async_api._api.dto.ask_page_dto import AskPageDTO +from dendrite.browser.async_api._api.dto.try_run_script_dto import TryRunScriptDTO +from dendrite.browser.async_api._api.response.selector_cache_response import SelectorCacheResponse +from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse +from dendrite.browser.async_api._api.response.interaction_response import InteractionResponse +from dendrite.browser.async_api._api.response.extract_response import ExtractResponse +from dendrite.browser.async_api._api.response.ask_page_response import AskPageResponse +from dendrite.browser.async_api._api.response.cache_extract_response import CacheExtractResponse +from dendrite.browser.async_api._core.models.authentication import AuthSession + + +class BrowserAPIProtocol(Protocol): + async def authenticate(self, dto: AuthenticateDTO) -> AuthSession: + ... + + async def upload_auth_session(self, dto: UploadAuthSessionDTO) -> None: + ... + + async def check_selector_cache(self, dto: CheckSelectorCacheDTO) -> SelectorCacheResponse: + ... + + async def get_interactions_selector(self, dto: GetElementsDTO) -> GetElementResponse: + ... + + async def make_interaction(self, dto: MakeInteractionDTO) -> InteractionResponse: + ... + + async def check_extract_cache(self, dto: ExtractDTO) -> CacheExtractResponse: + ... + + async def extract(self, dto: ExtractDTO) -> ExtractResponse: + ... + + async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: + ... + + async def try_run_cached(self, dto: TryRunScriptDTO) -> Optional[ExtractResponse]: + ... \ No newline at end of file diff --git a/dendrite/remote/__init__.py b/dendrite/remote/__init__.py deleted file mode 100644 index 434c45d..0000000 --- a/dendrite/remote/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from typing import Union -from dendrite.remote.browserless_config import BrowserlessConfig -from dendrite.remote.browserbase_config import BrowserbaseConfig - - -Providers = Union[BrowserbaseConfig, BrowserlessConfig] - -__all__ = ["Providers", "BrowserbaseConfig"] diff --git a/dendrite/sync_api/_api/dto/get_interaction_dto.py b/dendrite/sync_api/_api/dto/get_interaction_dto.py deleted file mode 100644 index bdc7654..0000000 --- a/dendrite/sync_api/_api/dto/get_interaction_dto.py +++ /dev/null @@ -1,9 +0,0 @@ -from pydantic import BaseModel -from dendrite.sync_api._core.models.api_config import APIConfig -from dendrite.sync_api._core.models.page_information import PageInformation - - -class GetInteractionDTO(BaseModel): - page_information: PageInformation - api_config: APIConfig - prompt: str diff --git a/scripts/generate_sync.py b/scripts/generate_sync.py index b6d69c0..b28b2df 100644 --- a/scripts/generate_sync.py +++ b/scripts/generate_sync.py @@ -138,7 +138,7 @@ def visit_Import(self, node): alias = ast.alias(name="time", asname=alias.asname) elif alias.name.startswith("dendrite"): new_name = alias.name.replace( - "dendrite.async_api", "dendrite.sync_api", 1 + "dendrite.browser.async_api", "dendrite.browser.sync_api", 1 ) alias = ast.alias(name=new_name, asname=alias.asname) new_names.append(alias) @@ -160,7 +160,7 @@ def visit_ImportFrom(self, node): node.module = "time" elif node.module and node.module.startswith("dendrite"): node.module = node.module.replace( - "dendrite.async_api", "dendrite.sync_api", 1 + "dendrite.browser.async_api", "dendrite.browser.sync_api", 1 ) return node diff --git a/tests/tests_async/conftest.py b/tests/tests_async/conftest.py index c74c2cd..1263fb6 100644 --- a/tests/tests_async/conftest.py +++ b/tests/tests_async/conftest.py @@ -2,10 +2,10 @@ import asyncio import pytest_asyncio -from dendrite.async_api._core.dendrite_browser import ( +from dendrite.browser.async_api._core.dendrite_browser import ( AsyncDendrite, ) -from dendrite.remote import ( +from dendrite.browser.remote import ( BrowserbaseConfig, ) # Import your class here diff --git a/tests/tests_async/test_browserbase.py b/tests/tests_async/test_browserbase.py index 616b3fd..9ab6d46 100644 --- a/tests/tests_async/test_browserbase.py +++ b/tests/tests_async/test_browserbase.py @@ -1,6 +1,6 @@ # import os # import pytest -# from dendrite.async_api._core.dendrite_browser import AsyncDendrite +# from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite # @pytest.mark.asyncio(loop_scope="session") diff --git a/tests/tests_async/test_download.py b/tests/tests_async/test_download.py index 608bef0..e4575fa 100644 --- a/tests/tests_async/test_download.py +++ b/tests/tests_async/test_download.py @@ -2,7 +2,7 @@ import os import pytest -from dendrite.async_api._core.dendrite_browser import AsyncDendrite +from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite pytest_plugins = ("pytest_asyncio",) diff --git a/tests/tests_sync/conftest.py b/tests/tests_sync/conftest.py index e7fbfc0..8fa6ae7 100644 --- a/tests/tests_sync/conftest.py +++ b/tests/tests_sync/conftest.py @@ -1,5 +1,5 @@ import pytest -from dendrite.sync_api import Dendrite +from dendrite.browser.sync_api import Dendrite @pytest.fixture(scope="session") diff --git a/tests/tests_sync/test_context.py b/tests/tests_sync/test_context.py index a1a0f4f..10a7dd6 100644 --- a/tests/tests_sync/test_context.py +++ b/tests/tests_sync/test_context.py @@ -1,6 +1,6 @@ # content of test_tmp_path.py import os -from dendrite.sync_api import Dendrite +from dendrite.browser.sync_api import Dendrite def test_context_manager(): diff --git a/tests/tests_sync/test_download.py b/tests/tests_sync/test_download.py index 4ddbe14..d3279bb 100644 --- a/tests/tests_sync/test_download.py +++ b/tests/tests_sync/test_download.py @@ -1,6 +1,6 @@ # content of test_tmp_path.py import os -from dendrite.sync_api import Dendrite +from dendrite.browser.sync_api import Dendrite def test_download(dendrite_browser: Dendrite, tmp_path): From 7a81b22d54bd32152cba87a6354fb3bd535c3bb9 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Mon, 18 Nov 2024 10:52:33 +0100 Subject: [PATCH 02/18] wip add local cache --- dendrite/__init__.py | 54 +- .../browser/_common/_exceptions/_constants.py | 2 +- .../{async_api => }/_common/constants.py | 0 .../_common/status.py => _common/types.py} | 1 - .../browser/async_api/_api/_http_client.py | 65 - .../async_api/_api/browser_api_client.py | 120 -- .../browser/async_api/_api/dto/__init__.py | 0 .../async_api/_api/dto/ask_page_dto.py | 11 - .../async_api/_api/dto/authenticate_dto.py | 6 - .../async_api/_api/dto/get_interaction_dto.py | 10 - .../async_api/_api/dto/get_session_dto.py | 7 - .../async_api/_api/dto/google_search_dto.py | 12 - .../_api/dto/make_interaction_dto.py | 19 - .../async_api/_api/dto/try_run_script_dto.py | 14 - .../_api/dto/upload_auth_session_dto.py | 11 - .../async_api/_api/response/__init__.py | 0 .../_api/response/ask_page_response.py | 11 - .../_api/response/cache_extract_response.py | 5 - .../_api/response/get_element_response.py | 12 - .../_api/response/google_search_response.py | 12 - .../_api/response/interaction_response.py | 7 - .../_api/response/selector_cache_response.py | 5 - .../_api/response/session_response.py | 7 - .../browser/async_api/_common/__init__.py | 0 .../browser/async_api/_core/_impl_browser.py | 24 +- .../browser/async_api/_core/_impl_mapping.py | 7 +- .../async_api/_core/_local_browser_impl.py | 29 + dendrite/browser/async_api/_core/_utils.py | 2 +- .../async_api/_core/dendrite_browser.py | 2 +- .../{_common => _core}/event_sync.py | 0 .../{_ext_impl => _remote_impl}/__init__.py | 0 .../browserbase/__init__.py | 0 .../browserbase/_client.py | 0 .../browserbase/_download.py | 2 +- .../browserbase/_impl.py | 4 +- .../browserless/__init__.py | 0 .../browserless/_impl.py | 4 +- .../browser/sync_api/_dom/util/mild_strip.py | 32 - .../cache/element_cache.py} | 0 dendrite/logic/cache/file_cache.py | 67 + dendrite/logic/factory.py | 17 + dendrite/logic/interfaces/cache.py | 24 + dendrite/logic/local/ask/ask.py | 237 +++ dendrite/logic/local/ask/image.py | 35 + dendrite/logic/local/code/code_session.py | 137 ++ dendrite/logic/local/code/execute.py | 26 + dendrite/logic/local/dom/css.py | 170 ++ .../local/dom/strip.py} | 0 dendrite/logic/local/dom/truncate.py | 72 + dendrite/logic/local/extract/extract_agent.py | 529 ++++++ dendrite/logic/local/extract/prompts.py | 230 +++ dendrite/logic/local/extract/scroll_agent.py | 246 +++ .../logic/local/get_element/agents/agent.py | 146 ++ .../get_element/agents/prompts/__init__.py | 12 + .../get_element/agents/prompts/segment.prompt | 107 ++ .../get_element/agents/prompts/select.prompt | 90 + .../local/get_element/agents/segment_agent.py | 144 ++ .../local/get_element/agents/select_agent.py | 108 ++ .../local/get_element/cached_selector.py | 56 + dendrite/logic/local/get_element/dom.py | 338 ++++ .../logic/local/get_element/hanifi_search.py | 193 ++ dendrite/logic/local/get_element/main.py | 101 + dendrite/logic/local/get_element/models.py | 16 + dendrite/logic/local/llm/token_count.py | 7 + dendrite/models/api_config.py | 33 + dendrite/models/dto/ask_page_dto.py | 12 + .../_api => models}/dto/extract_dto.py | 8 +- .../_api => models}/dto/get_elements_dto.py | 9 +- dendrite/models/page_information.py | 8 + .../response/extract_page_response.py} | 7 +- poetry.lock | 1662 ++++++++++++++++- pyproject.toml | 3 + 72 files changed, 4904 insertions(+), 443 deletions(-) rename dendrite/browser/{async_api => }/_common/constants.py (100%) rename dendrite/browser/{async_api/_common/status.py => _common/types.py} (98%) delete mode 100644 dendrite/browser/async_api/_api/_http_client.py delete mode 100644 dendrite/browser/async_api/_api/browser_api_client.py delete mode 100644 dendrite/browser/async_api/_api/dto/__init__.py delete mode 100644 dendrite/browser/async_api/_api/dto/ask_page_dto.py delete mode 100644 dendrite/browser/async_api/_api/dto/authenticate_dto.py delete mode 100644 dendrite/browser/async_api/_api/dto/get_interaction_dto.py delete mode 100644 dendrite/browser/async_api/_api/dto/get_session_dto.py delete mode 100644 dendrite/browser/async_api/_api/dto/google_search_dto.py delete mode 100644 dendrite/browser/async_api/_api/dto/make_interaction_dto.py delete mode 100644 dendrite/browser/async_api/_api/dto/try_run_script_dto.py delete mode 100644 dendrite/browser/async_api/_api/dto/upload_auth_session_dto.py delete mode 100644 dendrite/browser/async_api/_api/response/__init__.py delete mode 100644 dendrite/browser/async_api/_api/response/ask_page_response.py delete mode 100644 dendrite/browser/async_api/_api/response/cache_extract_response.py delete mode 100644 dendrite/browser/async_api/_api/response/get_element_response.py delete mode 100644 dendrite/browser/async_api/_api/response/google_search_response.py delete mode 100644 dendrite/browser/async_api/_api/response/interaction_response.py delete mode 100644 dendrite/browser/async_api/_api/response/selector_cache_response.py delete mode 100644 dendrite/browser/async_api/_api/response/session_response.py delete mode 100644 dendrite/browser/async_api/_common/__init__.py create mode 100644 dendrite/browser/async_api/_core/_local_browser_impl.py rename dendrite/browser/async_api/{_common => _core}/event_sync.py (100%) rename dendrite/browser/async_api/{_ext_impl => _remote_impl}/__init__.py (100%) rename dendrite/browser/async_api/{_ext_impl => _remote_impl}/browserbase/__init__.py (100%) rename dendrite/browser/async_api/{_ext_impl => _remote_impl}/browserbase/_client.py (100%) rename dendrite/browser/async_api/{_ext_impl => _remote_impl}/browserbase/_download.py (96%) rename dendrite/browser/async_api/{_ext_impl => _remote_impl}/browserbase/_impl.py (94%) rename dendrite/browser/async_api/{_ext_impl => _remote_impl}/browserless/__init__.py (100%) rename dendrite/browser/async_api/{_ext_impl => _remote_impl}/browserless/_impl.py (92%) delete mode 100644 dendrite/browser/sync_api/_dom/util/mild_strip.py rename dendrite/{browser/async_api/_api/__init__.py => logic/cache/element_cache.py} (100%) create mode 100644 dendrite/logic/cache/file_cache.py create mode 100644 dendrite/logic/factory.py create mode 100644 dendrite/logic/interfaces/cache.py create mode 100644 dendrite/logic/local/ask/ask.py create mode 100644 dendrite/logic/local/ask/image.py create mode 100644 dendrite/logic/local/code/code_session.py create mode 100644 dendrite/logic/local/code/execute.py create mode 100644 dendrite/logic/local/dom/css.py rename dendrite/{browser/async_api/_dom/util/mild_strip.py => logic/local/dom/strip.py} (100%) create mode 100644 dendrite/logic/local/dom/truncate.py create mode 100644 dendrite/logic/local/extract/extract_agent.py create mode 100644 dendrite/logic/local/extract/prompts.py create mode 100644 dendrite/logic/local/extract/scroll_agent.py create mode 100644 dendrite/logic/local/get_element/agents/agent.py create mode 100644 dendrite/logic/local/get_element/agents/prompts/__init__.py create mode 100644 dendrite/logic/local/get_element/agents/prompts/segment.prompt create mode 100644 dendrite/logic/local/get_element/agents/prompts/select.prompt create mode 100644 dendrite/logic/local/get_element/agents/segment_agent.py create mode 100644 dendrite/logic/local/get_element/agents/select_agent.py create mode 100644 dendrite/logic/local/get_element/cached_selector.py create mode 100644 dendrite/logic/local/get_element/dom.py create mode 100644 dendrite/logic/local/get_element/hanifi_search.py create mode 100644 dendrite/logic/local/get_element/main.py create mode 100644 dendrite/logic/local/get_element/models.py create mode 100644 dendrite/logic/local/llm/token_count.py create mode 100644 dendrite/models/api_config.py create mode 100644 dendrite/models/dto/ask_page_dto.py rename dendrite/{browser/async_api/_api => models}/dto/extract_dto.py (78%) rename dendrite/{browser/async_api/_api => models}/dto/get_elements_dto.py (65%) create mode 100644 dendrite/models/page_information.py rename dendrite/{browser/async_api/_api/response/extract_response.py => models/response/extract_page_response.py} (51%) diff --git a/dendrite/__init__.py b/dendrite/__init__.py index 1e5852c..310bd90 100644 --- a/dendrite/__init__.py +++ b/dendrite/__init__.py @@ -1,33 +1,33 @@ -import sys -from loguru import logger -from dendrite.browser.async_api import ( - AsyncDendrite, - AsyncElement, - AsyncPage, - AsyncElementsResponse, -) +# import sys +# from loguru import logger +# from dendrite.browser.async_api import ( +# AsyncDendrite, +# AsyncElement, +# AsyncPage, +# AsyncElementsResponse, +# ) -from dendrite.browser.sync_api import ( - Dendrite, - Element, - Page, - ElementsResponse, -) +# from dendrite.browser.sync_api import ( +# Dendrite, +# Element, +# Page, +# ElementsResponse, +# ) -logger.remove() +# logger.remove() -fmt = "{time: HH:mm:ss.SSS} | {level: <8}- {message}" +# fmt = "{time: HH:mm:ss.SSS} | {level: <8}- {message}" -logger.add(sys.stderr, level="INFO", format=fmt) +# logger.add(sys.stderr, level="INFO", format=fmt) -__all__ = [ - "AsyncDendrite", - "AsyncElement", - "AsyncPage", - "AsyncElementsResponse", - "Dendrite", - "Element", - "Page", - "ElementsResponse", -] +# __all__ = [ +# "AsyncDendrite", +# "AsyncElement", +# "AsyncPage", +# "AsyncElementsResponse", +# "Dendrite", +# "Element", +# "Page", +# "ElementsResponse", +# ] diff --git a/dendrite/browser/_common/_exceptions/_constants.py b/dendrite/browser/_common/_exceptions/_constants.py index f970308..a507e87 100644 --- a/dendrite/browser/_common/_exceptions/_constants.py +++ b/dendrite/browser/_common/_exceptions/_constants.py @@ -1 +1 @@ -INVALID_AUTH_SESSION_MSG = "Missing auth session for any of: {domain}. Make sure that you have used the Dendrite Vault extension to extract your authenticated session(s) for the domain(s) you are trying to access." +INVALID_AUTH_SESSION_MSG = "Missing auth session for any of: {domain}. Make sure that you have used the Dendrite Vault extension to extract your authenticated session(s) for the domain(s) you are trying to access." \ No newline at end of file diff --git a/dendrite/browser/async_api/_common/constants.py b/dendrite/browser/_common/constants.py similarity index 100% rename from dendrite/browser/async_api/_common/constants.py rename to dendrite/browser/_common/constants.py diff --git a/dendrite/browser/async_api/_common/status.py b/dendrite/browser/_common/types.py similarity index 98% rename from dendrite/browser/async_api/_common/status.py rename to dendrite/browser/_common/types.py index 0068d7d..427449d 100644 --- a/dendrite/browser/async_api/_common/status.py +++ b/dendrite/browser/_common/types.py @@ -1,4 +1,3 @@ from typing import Literal - Status = Literal["success", "failed", "loading", "impossible"] diff --git a/dendrite/browser/async_api/_api/_http_client.py b/dendrite/browser/async_api/_api/_http_client.py deleted file mode 100644 index 72777e2..0000000 --- a/dendrite/browser/async_api/_api/_http_client.py +++ /dev/null @@ -1,65 +0,0 @@ -import os -from typing import Optional -import httpx -from loguru import logger - -from dendrite.browser.async_api._core.models.api_config import APIConfig - - -class HTTPClient: - def __init__(self, api_config: APIConfig, session_id: Optional[str] = None): - self.api_key = api_config.dendrite_api_key - self.session_id = session_id - self.base_url = self.resolve_base_url() - - def resolve_base_url(self): - base_url = ( - "http://localhost:8000/api/v1" - if os.environ.get("DENDRITE_DEV") - else "https://dendrite-server.azurewebsites.net/api/v1" - ) - return base_url - - async def send_request( - self, - endpoint: str, - params: Optional[dict] = None, - data: Optional[dict] = None, - headers: Optional[dict] = None, - method: str = "GET", - ) -> httpx.Response: - url = f"{self.base_url}/{endpoint}" - - headers = headers or {} - headers["Content-Type"] = "application/json" - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - if self.session_id: - headers["X-Session-ID"] = self.session_id - - async with httpx.AsyncClient(timeout=300) as client: - try: - response = await client.request( - method, url, params=params, json=data, headers=headers - ) - response.raise_for_status() - # logger.debug( - # f"{method} to '{url}', that took: { time.time() - start_time }\n\nResponse: {dict_res}\n\n" - # ) - return response - except httpx.HTTPStatusError as http_err: - logger.debug( - f"HTTP error occurred: {http_err.response.status_code}: {http_err.response.text}" - ) - raise - except httpx.ConnectError as connect_err: - logger.error( - f"Connection error occurred: {connect_err}. {url} Server might be down" - ) - raise - except httpx.RequestError as req_err: - # logger.debug(f"Request error occurred: {req_err}") - raise - except Exception as err: - # logger.debug(f"An error occurred: {err}") - raise diff --git a/dendrite/browser/async_api/_api/browser_api_client.py b/dendrite/browser/async_api/_api/browser_api_client.py deleted file mode 100644 index b52738f..0000000 --- a/dendrite/browser/async_api/_api/browser_api_client.py +++ /dev/null @@ -1,120 +0,0 @@ -from typing import Optional - -from loguru import logger -from dendrite.browser.async_api._api.response.cache_extract_response import ( - CacheExtractResponse, -) -from dendrite.browser.async_api._api.response.selector_cache_response import ( - SelectorCacheResponse, -) -from dendrite.browser.async_api._core.models.authentication import AuthSession -from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse -from dendrite.browser.async_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.browser.async_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.browser.async_api._api.dto.make_interaction_dto import MakeInteractionDTO -from dendrite.browser.async_api._api.dto.extract_dto import ExtractDTO -from dendrite.browser.async_api._api.dto.try_run_script_dto import TryRunScriptDTO -from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.browser.async_api._api.response.ask_page_response import AskPageResponse -from dendrite.browser.async_api._api.response.interaction_response import ( - InteractionResponse, -) -from dendrite.browser.async_api._api.response.extract_response import ExtractResponse -from dendrite.browser.async_api._api._http_client import HTTPClient -from dendrite.browser._common._exceptions.dendrite_exception import ( - InvalidAuthSessionError, -) -from dendrite.browser.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO - - -class BrowserAPIClient(HTTPClient): - - async def authenticate(self, dto: AuthenticateDTO): - res = await self.send_request( - "actions/authenticate", data=dto.model_dump(), method="POST" - ) - - if res.status_code == 204: - raise InvalidAuthSessionError(domain=dto.domains) - - return AuthSession(**res.json()) - - async def upload_auth_session(self, dto: UploadAuthSessionDTO): - await self.send_request( - "actions/upload-auth-session", data=dto.dict(), method="POST" - ) - - async def check_selector_cache( - self, dto: CheckSelectorCacheDTO - ) -> SelectorCacheResponse: - res = await self.send_request( - "actions/check-selector-cache", data=dto.dict(), method="POST" - ) - return SelectorCacheResponse(**res.json()) - - async def get_interactions_selector( - self, dto: GetElementsDTO - ) -> GetElementResponse: - res = await self.send_request( - "actions/get-interaction-selector", data=dto.dict(), method="POST" - ) - return GetElementResponse(**res.json()) - - async def make_interaction(self, dto: MakeInteractionDTO) -> InteractionResponse: - res = await self.send_request( - "actions/make-interaction", data=dto.dict(), method="POST" - ) - res_dict = res.json() - return InteractionResponse( - status=res_dict["status"], message=res_dict["message"] - ) - - async def check_extract_cache(self, dto: ExtractDTO) -> CacheExtractResponse: - res = await self.send_request( - "actions/check-extract-cache", data=dto.dict(), method="POST" - ) - return CacheExtractResponse(**res.json()) - - async def extract(self, dto: ExtractDTO) -> ExtractResponse: - res = await self.send_request( - "actions/extract-page", data=dto.dict(), method="POST" - ) - res_dict = res.json() - return ExtractResponse( - status=res_dict["status"], - message=res_dict["message"], - return_data=res_dict["return_data"], - created_script=res_dict.get("created_script", None), - used_cache=res_dict.get("used_cache", False), - ) - - async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: - res = await self.send_request( - "actions/ask-page", data=dto.dict(), method="POST" - ) - res_dict = res.json() - return AskPageResponse( - status=res_dict["status"], - description=res_dict["description"], - return_data=res_dict["return_data"], - ) - - async def try_run_cached(self, dto: TryRunScriptDTO) -> Optional[ExtractResponse]: - res = await self.send_request( - "actions/try-run-cached", data=dto.dict(), method="POST" - ) - if res is None: - return None - res_dict = res.json() - loaded_value = res_dict["return_data"] - if loaded_value is None: - return None - - return ExtractResponse( - status=res_dict["status"], - message=res_dict["message"], - return_data=loaded_value, - created_script=res_dict.get("created_script", None), - used_cache=res_dict.get("used_cache", False), - ) diff --git a/dendrite/browser/async_api/_api/dto/__init__.py b/dendrite/browser/async_api/_api/dto/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/async_api/_api/dto/ask_page_dto.py b/dendrite/browser/async_api/_api/dto/ask_page_dto.py deleted file mode 100644 index f2dcaf3..0000000 --- a/dendrite/browser/async_api/_api/dto/ask_page_dto.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Any, Optional -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class AskPageDTO(BaseModel): - prompt: str - return_schema: Optional[Any] - page_information: PageInformation - api_config: APIConfig diff --git a/dendrite/browser/async_api/_api/dto/authenticate_dto.py b/dendrite/browser/async_api/_api/dto/authenticate_dto.py deleted file mode 100644 index f5a1de7..0000000 --- a/dendrite/browser/async_api/_api/dto/authenticate_dto.py +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Union -from pydantic import BaseModel - - -class AuthenticateDTO(BaseModel): - domains: Union[str, list[str]] diff --git a/dendrite/browser/async_api/_api/dto/get_interaction_dto.py b/dendrite/browser/async_api/_api/dto/get_interaction_dto.py deleted file mode 100644 index 93889c7..0000000 --- a/dendrite/browser/async_api/_api/dto/get_interaction_dto.py +++ /dev/null @@ -1,10 +0,0 @@ -from pydantic import BaseModel - -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class GetInteractionDTO(BaseModel): - page_information: PageInformation - api_config: APIConfig - prompt: str diff --git a/dendrite/browser/async_api/_api/dto/get_session_dto.py b/dendrite/browser/async_api/_api/dto/get_session_dto.py deleted file mode 100644 index 6414cc3..0000000 --- a/dendrite/browser/async_api/_api/dto/get_session_dto.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class GetSessionDTO(BaseModel): - user_id: str - domain: str diff --git a/dendrite/browser/async_api/_api/dto/google_search_dto.py b/dendrite/browser/async_api/_api/dto/google_search_dto.py deleted file mode 100644 index 6c55615..0000000 --- a/dendrite/browser/async_api/_api/dto/google_search_dto.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Optional -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class GoogleSearchDTO(BaseModel): - query: str - country: Optional[str] = None - filter_results_prompt: Optional[str] = None - page_information: PageInformation - api_config: APIConfig diff --git a/dendrite/browser/async_api/_api/dto/make_interaction_dto.py b/dendrite/browser/async_api/_api/dto/make_interaction_dto.py deleted file mode 100644 index c0592ad..0000000 --- a/dendrite/browser/async_api/_api/dto/make_interaction_dto.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Literal, Optional -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_diff_information import ( - PageDiffInformation, -) - - -InteractionType = Literal["click", "fill", "hover"] - - -class MakeInteractionDTO(BaseModel): - url: str - dendrite_id: str - interaction_type: InteractionType - value: Optional[str] = None - expected_outcome: Optional[str] - page_delta_information: PageDiffInformation - api_config: APIConfig diff --git a/dendrite/browser/async_api/_api/dto/try_run_script_dto.py b/dendrite/browser/async_api/_api/dto/try_run_script_dto.py deleted file mode 100644 index e283806..0000000 --- a/dendrite/browser/async_api/_api/dto/try_run_script_dto.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import Any, Optional -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig - - -class TryRunScriptDTO(BaseModel): - url: str - raw_html: str - api_config: APIConfig - prompt: str - db_prompt: Optional[str] = ( - None # If you wish to cache a script based of a fixed prompt use this value - ) - return_data_json_schema: Any diff --git a/dendrite/browser/async_api/_api/dto/upload_auth_session_dto.py b/dendrite/browser/async_api/_api/dto/upload_auth_session_dto.py deleted file mode 100644 index 1697fdf..0000000 --- a/dendrite/browser/async_api/_api/dto/upload_auth_session_dto.py +++ /dev/null @@ -1,11 +0,0 @@ -from pydantic import BaseModel - -from dendrite.browser.async_api._core.models.authentication import ( - AuthSession, - StorageState, -) - - -class UploadAuthSessionDTO(BaseModel): - auth_data: AuthSession - storage_state: StorageState diff --git a/dendrite/browser/async_api/_api/response/__init__.py b/dendrite/browser/async_api/_api/response/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/async_api/_api/response/ask_page_response.py b/dendrite/browser/async_api/_api/response/ask_page_response.py deleted file mode 100644 index 4ec747a..0000000 --- a/dendrite/browser/async_api/_api/response/ask_page_response.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Generic, Literal, TypeVar -from pydantic import BaseModel - - -T = TypeVar("T") - - -class AskPageResponse(BaseModel, Generic[T]): - status: Literal["success", "error"] - return_data: T - description: str diff --git a/dendrite/browser/async_api/_api/response/cache_extract_response.py b/dendrite/browser/async_api/_api/response/cache_extract_response.py deleted file mode 100644 index 463d03b..0000000 --- a/dendrite/browser/async_api/_api/response/cache_extract_response.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class CacheExtractResponse(BaseModel): - exists: bool diff --git a/dendrite/browser/async_api/_api/response/get_element_response.py b/dendrite/browser/async_api/_api/response/get_element_response.py deleted file mode 100644 index 8fb8fa5..0000000 --- a/dendrite/browser/async_api/_api/response/get_element_response.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Dict, List, Optional, Union - -from pydantic import BaseModel - -from dendrite.browser.async_api._common.status import Status - - -class GetElementResponse(BaseModel): - status: Status - selectors: Optional[Union[List[str], Dict[str, List[str]]]] = None - message: str = "" - used_cache: bool = False diff --git a/dendrite/browser/async_api/_api/response/google_search_response.py b/dendrite/browser/async_api/_api/response/google_search_response.py deleted file mode 100644 index d435b71..0000000 --- a/dendrite/browser/async_api/_api/response/google_search_response.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class SearchResult(BaseModel): - url: str - title: str - description: str - - -class GoogleSearchResponse(BaseModel): - results: List[SearchResult] diff --git a/dendrite/browser/async_api/_api/response/interaction_response.py b/dendrite/browser/async_api/_api/response/interaction_response.py deleted file mode 100644 index 3dd2e49..0000000 --- a/dendrite/browser/async_api/_api/response/interaction_response.py +++ /dev/null @@ -1,7 +0,0 @@ -from pydantic import BaseModel -from dendrite.browser.async_api._common.status import Status - - -class InteractionResponse(BaseModel): - message: str - status: Status diff --git a/dendrite/browser/async_api/_api/response/selector_cache_response.py b/dendrite/browser/async_api/_api/response/selector_cache_response.py deleted file mode 100644 index 4c0e388..0000000 --- a/dendrite/browser/async_api/_api/response/selector_cache_response.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class SelectorCacheResponse(BaseModel): - exists: bool diff --git a/dendrite/browser/async_api/_api/response/session_response.py b/dendrite/browser/async_api/_api/response/session_response.py deleted file mode 100644 index 2d03b97..0000000 --- a/dendrite/browser/async_api/_api/response/session_response.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class SessionResponse(BaseModel): - cookies: List[dict] - origins_storage: List[dict] diff --git a/dendrite/browser/async_api/_common/__init__.py b/dendrite/browser/async_api/_common/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/async_api/_core/_impl_browser.py b/dendrite/browser/async_api/_core/_impl_browser.py index b01f313..97709a4 100644 --- a/dendrite/browser/async_api/_core/_impl_browser.py +++ b/dendrite/browser/async_api/_core/_impl_browser.py @@ -63,26 +63,4 @@ async def stop_session(self) -> None: Raises: Exception: If there is an issue stopping the browser session. """ - pass - - -class LocalImpl(ImplBrowser): - def __init__(self) -> None: - pass - - async def start_browser(self, playwright: Playwright, pw_options) -> Browser: - return await playwright.chromium.launch(**pw_options) - - async def get_download( - self, - dendrite_browser: "AsyncDendrite", - pw_page: PlaywrightPage, - timeout: float, - ) -> Download: - return await dendrite_browser._download_handler.get_data(pw_page, timeout) - - async def configure_context(self, browser: "AsyncDendrite"): - pass - - async def stop_session(self): - pass + pass \ No newline at end of file diff --git a/dendrite/browser/async_api/_core/_impl_mapping.py b/dendrite/browser/async_api/_core/_impl_mapping.py index b68292a..7022aa5 100644 --- a/dendrite/browser/async_api/_core/_impl_mapping.py +++ b/dendrite/browser/async_api/_core/_impl_mapping.py @@ -1,9 +1,10 @@ from typing import Any, Dict, Optional, Type -from dendrite.browser.async_api._core._impl_browser import ImplBrowser, LocalImpl +from dendrite.browser.async_api._core._impl_browser import ImplBrowser -from dendrite.browser.async_api._ext_impl.browserbase._impl import BrowserBaseImpl -from dendrite.browser.async_api._ext_impl.browserless._impl import BrowserlessImpl +from dendrite.browser.async_api._core._local_browser_impl import LocalImpl +from dendrite.browser.async_api._remote_impl.browserbase._impl import BrowserBaseImpl +from dendrite.browser.async_api._remote_impl.browserless._impl import BrowserlessImpl from dendrite.browser.remote.browserless_config import BrowserlessConfig from dendrite.browser.remote.browserbase_config import BrowserbaseConfig from dendrite.browser.remote import Providers diff --git a/dendrite/browser/async_api/_core/_local_browser_impl.py b/dendrite/browser/async_api/_core/_local_browser_impl.py new file mode 100644 index 0000000..61c0256 --- /dev/null +++ b/dendrite/browser/async_api/_core/_local_browser_impl.py @@ -0,0 +1,29 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + +from dendrite.browser.async_api._core._impl_browser import ImplBrowser +from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from playwright.async_api import Download, Browser, Playwright + +class LocalImpl(ImplBrowser): + def __init__(self) -> None: + pass + + async def start_browser(self, playwright: Playwright, pw_options) -> Browser: + return await playwright.chromium.launch(**pw_options) + + async def get_download( + self, + dendrite_browser: "AsyncDendrite", + pw_page: PlaywrightPage, + timeout: float, + ) -> Download: + return await dendrite_browser._download_handler.get_data(pw_page, timeout) + + async def configure_context(self, browser: "AsyncDendrite"): + pass + + async def stop_session(self): + pass \ No newline at end of file diff --git a/dendrite/browser/async_api/_core/_utils.py b/dendrite/browser/async_api/_core/_utils.py index 8860b08..6fd7142 100644 --- a/dendrite/browser/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_core/_utils.py @@ -14,7 +14,7 @@ from dendrite.browser.async_api._core._js import ( GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, ) -from dendrite.browser.async_api._dom.util.mild_strip import mild_strip_in_place +from dendrite.browser.async_api._dom.mild_strip import mild_strip_in_place async def expand_iframes( diff --git a/dendrite/browser/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/_core/dendrite_browser.py index d41c251..9abec39 100644 --- a/dendrite/browser/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/_core/dendrite_browser.py @@ -17,7 +17,7 @@ from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.browser.async_api._common.event_sync import EventSync +from dendrite.browser.async_api._core.event_sync import EventSync from dendrite.browser.async_api._core._impl_browser import ImplBrowser from dendrite.browser.async_api._core._impl_mapping import get_impl from dendrite.browser.async_api._core._managers.page_manager import ( diff --git a/dendrite/browser/async_api/_common/event_sync.py b/dendrite/browser/async_api/_core/event_sync.py similarity index 100% rename from dendrite/browser/async_api/_common/event_sync.py rename to dendrite/browser/async_api/_core/event_sync.py diff --git a/dendrite/browser/async_api/_ext_impl/__init__.py b/dendrite/browser/async_api/_remote_impl/__init__.py similarity index 100% rename from dendrite/browser/async_api/_ext_impl/__init__.py rename to dendrite/browser/async_api/_remote_impl/__init__.py diff --git a/dendrite/browser/async_api/_ext_impl/browserbase/__init__.py b/dendrite/browser/async_api/_remote_impl/browserbase/__init__.py similarity index 100% rename from dendrite/browser/async_api/_ext_impl/browserbase/__init__.py rename to dendrite/browser/async_api/_remote_impl/browserbase/__init__.py diff --git a/dendrite/browser/async_api/_ext_impl/browserbase/_client.py b/dendrite/browser/async_api/_remote_impl/browserbase/_client.py similarity index 100% rename from dendrite/browser/async_api/_ext_impl/browserbase/_client.py rename to dendrite/browser/async_api/_remote_impl/browserbase/_client.py diff --git a/dendrite/browser/async_api/_ext_impl/browserbase/_download.py b/dendrite/browser/async_api/_remote_impl/browserbase/_download.py similarity index 96% rename from dendrite/browser/async_api/_ext_impl/browserbase/_download.py rename to dendrite/browser/async_api/_remote_impl/browserbase/_download.py index dc355c3..965e932 100644 --- a/dendrite/browser/async_api/_ext_impl/browserbase/_download.py +++ b/dendrite/browser/async_api/_remote_impl/browserbase/_download.py @@ -7,7 +7,7 @@ from playwright.async_api import Download from dendrite.browser.async_api._core.models.download_interface import DownloadInterface -from dendrite.browser.async_api._ext_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.async_api._remote_impl.browserbase._client import BrowserbaseClient class AsyncBrowserbaseDownload(DownloadInterface): diff --git a/dendrite/browser/async_api/_ext_impl/browserbase/_impl.py b/dendrite/browser/async_api/_remote_impl/browserbase/_impl.py similarity index 94% rename from dendrite/browser/async_api/_ext_impl/browserbase/_impl.py rename to dendrite/browser/async_api/_remote_impl/browserbase/_impl.py index d8eb1be..04dc3ec 100644 --- a/dendrite/browser/async_api/_ext_impl/browserbase/_impl.py +++ b/dendrite/browser/async_api/_remote_impl/browserbase/_impl.py @@ -6,11 +6,11 @@ if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.browser.async_api._ext_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.async_api._remote_impl.browserbase._client import BrowserbaseClient from playwright.async_api import Playwright from loguru import logger -from dendrite.browser.async_api._ext_impl.browserbase._download import ( +from dendrite.browser.async_api._remote_impl.browserbase._download import ( AsyncBrowserbaseDownload, ) diff --git a/dendrite/browser/async_api/_ext_impl/browserless/__init__.py b/dendrite/browser/async_api/_remote_impl/browserless/__init__.py similarity index 100% rename from dendrite/browser/async_api/_ext_impl/browserless/__init__.py rename to dendrite/browser/async_api/_remote_impl/browserless/__init__.py diff --git a/dendrite/browser/async_api/_ext_impl/browserless/_impl.py b/dendrite/browser/async_api/_remote_impl/browserless/_impl.py similarity index 92% rename from dendrite/browser/async_api/_ext_impl/browserless/_impl.py rename to dendrite/browser/async_api/_remote_impl/browserless/_impl.py index 1ee4eb7..625fcd0 100644 --- a/dendrite/browser/async_api/_ext_impl/browserless/_impl.py +++ b/dendrite/browser/async_api/_remote_impl/browserless/_impl.py @@ -7,12 +7,12 @@ if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.browser.async_api._ext_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.async_api._remote_impl.browserbase._client import BrowserbaseClient from playwright.async_api import Playwright from loguru import logger import urllib.parse -from dendrite.browser.async_api._ext_impl.browserbase._download import ( +from dendrite.browser.async_api._remote_impl.browserbase._download import ( AsyncBrowserbaseDownload, ) diff --git a/dendrite/browser/sync_api/_dom/util/mild_strip.py b/dendrite/browser/sync_api/_dom/util/mild_strip.py deleted file mode 100644 index 7cd6923..0000000 --- a/dendrite/browser/sync_api/_dom/util/mild_strip.py +++ /dev/null @@ -1,32 +0,0 @@ -from bs4 import BeautifulSoup, Doctype, Tag, Comment - - -def mild_strip(soup: Tag, keep_d_id: bool = True) -> BeautifulSoup: - new_soup = BeautifulSoup(str(soup), "html.parser") - _mild_strip(new_soup, keep_d_id) - return new_soup - - -def mild_strip_in_place(soup: BeautifulSoup, keep_d_id: bool = True) -> None: - _mild_strip(soup, keep_d_id) - - -def _mild_strip(soup: BeautifulSoup, keep_d_id: bool = True) -> None: - for element in soup(text=lambda text: isinstance(text, Comment)): - element.extract() - for tag in soup( - ["head", "script", "style", "path", "polygon", "defs", "svg", "br", "Doctype"] - ): - tag.extract() - for element in soup.contents: - if isinstance(element, Doctype): - element.extract() - for tag in soup.find_all(True): - if tag.attrs.get("is-interactable-d_id") == "true": - continue - tag.attrs = { - attr: value[:100] if isinstance(value, str) else value - for (attr, value) in tag.attrs.items() - } - if keep_d_id == False: - del tag["d-id"] diff --git a/dendrite/browser/async_api/_api/__init__.py b/dendrite/logic/cache/element_cache.py similarity index 100% rename from dendrite/browser/async_api/_api/__init__.py rename to dendrite/logic/cache/element_cache.py diff --git a/dendrite/logic/cache/file_cache.py b/dendrite/logic/cache/file_cache.py new file mode 100644 index 0000000..f4bcf61 --- /dev/null +++ b/dendrite/logic/cache/file_cache.py @@ -0,0 +1,67 @@ +from pathlib import Path +import json +import threading +from typing import Generic, TypeVar, Union, Type, Dict +from pydantic import BaseModel +from hashlib import md5 + +T = TypeVar('T', bound=BaseModel) + +class FileCache(Generic[T]): + def __init__(self, model_class: Type[T], filepath: str = './cache.json'): + self.filepath = Path(filepath) + self.model_class = model_class + self.lock = threading.RLock() + self.cache: Dict[str, T] = {} + + # Create file if it doesn't exist + if not self.filepath.exists(): + self.filepath.parent.mkdir(parents=True, exist_ok=True) + self._save_cache({}) + else: + self._load_cache() + + def _load_cache(self) -> None: + """Load cache from file into memory""" + with self.lock: + try: + json_string = self.filepath.read_text() + raw_dict = json.loads(json_string) + + # Convert each entry back to the model class + self.cache = { + k: self.model_class.model_validate_json(json.dumps(v)) + for k, v in raw_dict.items() + } + except (json.JSONDecodeError, FileNotFoundError): + self.cache = {} + + def _save_cache(self, cache_dict: Dict[str, T]) -> None: + """Save cache to file""" + with self.lock: + # Convert models to dict before saving + serializable_dict = { + k: json.loads(v.model_dump_json()) + for k, v in cache_dict.items() + } + self.filepath.write_text(json.dumps(serializable_dict, indent=2)) + + def get(self, key: str) -> Union[T, None]: + hashed_key = self.hash(key) + return self.cache.get(hashed_key) + + def set(self, key: str, value: T) -> None: + hashed_key = self.hash(key) + with self.lock: + self.cache[hashed_key] = value + self._save_cache(self.cache) + + def delete(self, key: str) -> None: + hashed_key = self.hash(key) + with self.lock: + if hashed_key in self.cache: + del self.cache[hashed_key] + self._save_cache(self.cache) + + def hash(self, key: str) -> str: + return md5(key.encode()).hexdigest() diff --git a/dendrite/logic/factory.py b/dendrite/logic/factory.py new file mode 100644 index 0000000..542ae4f --- /dev/null +++ b/dendrite/logic/factory.py @@ -0,0 +1,17 @@ + +from typing import Literal, Optional + +from dendrite.logic.interfaces.async_api import BrowserAPIProtocol + + +class BrowserAPIFactory: + @staticmethod + def create_browser_api( + mode: Literal["local", "remote"], + api_config: APIConfig, + session_id: Optional[str] = None + ) -> BrowserAPIProtocol: + if mode == "local": + return LocalBrowserAPI() + else: + return BrowserAPIClient(api_config, session_id) \ No newline at end of file diff --git a/dendrite/logic/interfaces/cache.py b/dendrite/logic/interfaces/cache.py new file mode 100644 index 0000000..46ba571 --- /dev/null +++ b/dendrite/logic/interfaces/cache.py @@ -0,0 +1,24 @@ + +from typing import Protocol, Union, overload + +from typing import Protocol, TypeVar, Generic +from pydantic import BaseModel + +T = TypeVar('T', bound=BaseModel) + +class CacheProtocol(Protocol, Generic[T]): + + @overload + def get(self, key: dict) -> Union[T, None]: + ... + @overload + def get(self, key: str) -> Union[T, None]: + ... + def get(self, key: Union[str,dict]) -> Union[T, None]: + ... + + def set(self, key: str, value: T) -> None: + ... + + def delete(self, key: str) -> None: + ... \ No newline at end of file diff --git a/dendrite/logic/local/ask/ask.py b/dendrite/logic/local/ask/ask.py new file mode 100644 index 0000000..7464826 --- /dev/null +++ b/dendrite/logic/local/ask/ask.py @@ -0,0 +1,237 @@ +import json +import re +from anthropic.types import TextBlock +from jsonschema import validate +import json_repair + +from .image import segment_image + + +from dendrite_server_merge.dto.AskPageDTO import AskPageDTO +from dendrite_server_merge.responses.AskPageResponse import AskPageResponse + + +async def ask_page_action( + ask_page_dto: AskPageDTO, +) -> AskPageResponse: + image_segments = segment_image( + ask_page_dto.page_information.screenshot_base64, segment_height=2000 + ) + + scrolled_to_segment_i = 0 + content = generate_ask_page_prompt(ask_page_dto, image_segments) + messages = [{"role": "user", "content": content},] + + max_iterations = len(image_segments) + 5 + iteration = 0 + while iteration < max_iterations: + iteration += 1 + config = { + "messages": messages, + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.3, + "max_tokens": 1500, + } + res = await async_claude_request(config, ask_page_dto.api_config) + + if not isinstance(res.content[0], TextBlock): + raise Exception("Needs to be an text block") + + text = res.content[0].text + dict_res = { + "role": "assistant", + "content": text, + } + messages.append(dict_res) + + json_pattern = r"```json(.*?)```" + + if not text: + continue + + json_matches = re.findall(json_pattern, text, re.DOTALL) + + if len(json_matches) == 0: + continue + + extracted_json = json_matches[0].strip() + data_dict = json_repair.loads(extracted_json) + + if not isinstance(data_dict, dict): + content = "Your message doesn't contain a correctly formatted json object, try again." + messages.append({"role": "user","content": content}) + continue + + if "scroll_down" in data_dict: + next = scrolled_to_segment_i + 1 + if next < len(image_segments): + content = generate_scroll_prompt(image_segments, next) + else: + content = "You cannot scroll any further." + messages.append({"role": "user", "content": content}) + continue + + elif "return_data" in data_dict and "description" in data_dict: + return_data = data_dict["return_data"] + try: + if ask_page_dto.return_schema: + validate( + instance=return_data, schema=ask_page_dto.return_schema + ) + except Exception as e: + err_message = "Your return data doesn't match the requested return json schema, try again. Exception: {e}" + messages.append( + { + "role": "user", + "content": err_message, + } + ) + continue + + return AskPageResponse( + status="success", + return_data=data_dict["return_data"], + description=data_dict["description"], + ) + + elif "error" in data_dict: + was_blocked = data_dict.get("was_blocked_by_recaptcha", False) + return AskPageResponse( + status="error", + return_data=data_dict["error"], + description=f'{data_dict["error"]}, was_blocked_by_recaptcha: {was_blocked}', + ) + + else: + err_message = "Your message doesn't contain a correctly formatted action, try again." + messages.append( + { + "role": "user", + "content": err_message, + } + ) + + return AskPageResponse( + status="error", + return_data="Scrolled through the entire page without finding the requested data.", + description="", + ) + + + + + +def generate_ask_page_prompt(ask_page_dto: AskPageDTO, image_segments: list, scrolled_to_segment_i: int = 0) -> list: + # Generate scroll down hint based on number of segments + scroll_down_hint = ( + "" + if len(image_segments) == 1 + else """ + +If you think need to scroll further down, output an object with the key scroll down and nothing else: + +Action Message: +[Short reasoning first] +```json +{ + "scroll_down": true +} +``` + +You can keep scrolling down, noting important details, until you are ready to return the requested data, which you would do in a separate message.""" + ) + + # Get return schema prompt + return_schema_prompt = ( + str(ask_page_dto.return_schema) + if ask_page_dto.return_schema + else "No schema specified by the user" + ) + + # Construct the main prompt content + content = [ + { + "type": "text", + "text": f"""Please look at the page and return data that matches the requested schema and prompt. + + +{ask_page_dto.prompt} + + + +{return_schema_prompt} + + +Look the viewport and decide on the next action: + +If you can solve the prompt and return the requested data from the viewport, output a message with tripple backticks and 'json' like in the example below. Make sure `return_data` matches the requested return schema: + +Action Message: +[Short reasoning first] +```json +{{ + "description": "E.g There is a red button with the text 'get started' positoned underneath the title 'welcome!'", + "return_data": {{"element_exists": true, "foo": "bar"}}, +}} +``` + +Remember, `return_data` should be json that matches the structure of the requested json schema if available. Don't forget to include a description.{scroll_down_hint} + +In case you think the data is not available on the current page and the task does not describe how to handle the non-available data, or the page is blocked by a captcha puzzle or similar, output a json with a short error message, like this: + +Action Message: +[Short reasoning first.] +```json +{{ + "error": "reason why the task cannot be completed here", + "was_blocked_by_recaptcha": true/false +}} +``` + +Here is a screenshot of the viewport:""", + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": image_segments[scrolled_to_segment_i], + }, + }, + ] + + return content + +def generate_scroll_prompt(image_segments: list, next_segment: int) -> list: + """ + Generates the prompt for scrolling to next segment. + + Args: + image_segments: List of image segments + next_segment: Index of next segment + + Returns: + List of message content blocks + """ + last_segment_reminder = ( + " You won't be able to scroll further now." + if next_segment == len(image_segments) - 1 + else "" + ) + + content = [ + { + "type": "text", + "text": f"""You have scrolled down. You are viewing segment {next_segment+1}/{len(image_segments)}.{last_segment_reminder} Here is the new viewport:""", + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": image_segments[next_segment], + }, + }, + ] + + return content diff --git a/dendrite/logic/local/ask/image.py b/dendrite/logic/local/ask/image.py new file mode 100644 index 0000000..a8681ff --- /dev/null +++ b/dendrite/logic/local/ask/image.py @@ -0,0 +1,35 @@ +import io +import base64 +from typing import List +from PIL import Image + +from loguru import logger + + +def segment_image( + base64_image: str, + segment_height: int = 7900, +) -> List[str]: + if len(base64_image) < 100: + raise Exception("Failed to segment image since it is too small / glitched.") + + image_data = base64.b64decode(base64_image) + image = Image.open(io.BytesIO(image_data)) + width, height = image.size + segments = [] + + for i in range(0, height, segment_height): + # Define the box for cropping (left, upper, right, lower) + box = (0, i, width, min(i + segment_height, height)) + segment = image.crop(box) + + # Convert RGBA to RGB if necessary + if segment.mode == "RGBA": + segment = segment.convert("RGB") + + buffer = io.BytesIO() + segment.save(buffer, format="JPEG") + segment_data = buffer.getvalue() + segments.append(base64.b64encode(segment_data).decode()) + + return segments diff --git a/dendrite/logic/local/code/code_session.py b/dendrite/logic/local/code/code_session.py new file mode 100644 index 0000000..e6458c0 --- /dev/null +++ b/dendrite/logic/local/code/code_session.py @@ -0,0 +1,137 @@ +import sys +import traceback +import re # Important to keep since it is used inside the scripts +import json # Important to keep since it is used inside the scripts +from datetime import datetime # Important to keep since it is used inside the scripts + +from typing import Any, List, Optional +from bs4 import BeautifulSoup +from loguru import logger +from ..dom.truncate import truncate_long_string +from jsonschema import validate + + +class InterpreterError(Exception): + pass + + +def custom_exec( + cmd, + globals=None, + locals=None, +): + try: + exec(cmd, globals, locals) + except SyntaxError as err: + error_class = err.__class__.__name__ + detail = err.args[0] + line_number = err.lineno + except Exception as err: + error_class = err.__class__.__name__ + detail = err.args[0] + cl, exc, tb = sys.exc_info() + line_number = traceback.extract_tb(tb)[-1][1] + else: + return + + traceback_desc = traceback.format_exc() + raise InterpreterError( + f"{error_class} at line {line_number}. Detail: {detail}. Exception: {traceback_desc}" + ) + + +class CodeSession: + def __init__(self): + self.local_vars = {"soup": None, "html_string": "", "datetime": datetime} + + def get_local_var(self, name: str) -> Any: + try: + return self.local_vars[name] + except Exception as e: + return f"Error: Couldn't get local var with name {name}. Exception: {e}" + + def add_variable(self, name: str, value: Any): + self.local_vars[name] = value + + def exec_code( + self, + code: str, + soup: Optional[BeautifulSoup] = None, + html_string: Optional[str] = None, + ): + try: + self.local_vars["soup"] = soup + self.local_vars["html_string"] = html_string + self.local_vars["datetime"] = datetime + + copied_vars = self.local_vars.copy() + + try: + exec(code, globals(), copied_vars) + except SyntaxError as err: + error_class = err.__class__.__name__ + detail = err.args[0] + line_number = err.lineno + raise InterpreterError( + "%s at line %d, detail: %s" % (error_class, line_number, detail) + ) + except Exception as err: + error_class = err.__class__.__name__ + detail = err.args[0] + _, _, tb = sys.exc_info() + line_number = traceback.extract_tb(tb)[-1][1] + traceback_desc = traceback.format_exc() + raise InterpreterError( + "%s at line %d, detail: %s" + % (error_class, line_number, traceback_desc) + ) + + created_vars = { + k: v for k, v in copied_vars.items() if k not in self.local_vars + } + + self.local_vars = copied_vars + return created_vars + + except Exception as e: + raise Exception(f"Code failed to run. Exception: {e}") + + def validate_response(self, return_data_json_schema: Any, response_data: Any): + if return_data_json_schema != None: + try: + validate( + instance=response_data, + schema=return_data_json_schema, + ) + except Exception as e: + raise e + + def llm_readable_exec_res( + self, variables, prompt: str, attempts: int, max_attempts: int + ): + response = "Code executed.\n\n" + + if len(variables) == 0: + response += "No new variables were created." + else: + response += "Newly created variables:" + for var_name, var_value in variables.items(): + show_length = 600 if var_name == "response_data" else 300 + try: + # Convert var_value to string, handling potential errors + str_value = str(var_value) + except Exception as e: + logger.error(f"Error converting to string for display: {e}") + str_value = f"" + + truncated = truncate_long_string( + str_value, max_len_end=show_length, max_len_start=show_length + ) + extra_info = "" + if isinstance(var_value, List): + extra_info = f"\n{var_name}'s length is {len(var_value)}." + response += f"\n\n`{var_name}={truncated}`{extra_info}" + + response += f"\n\nDo these variables match the expected values? Remember, this is what the user asked for:\n\n{prompt}\n\nIf not, try again and remember, if one approach fails several times you might need to reinspect the DOM and try a different approach. You have {max_attempts - attempts} attempts left to try and complete the task. If you are happy with the results, output a success message." + + return response diff --git a/dendrite/logic/local/code/execute.py b/dendrite/logic/local/code/execute.py new file mode 100644 index 0000000..78720ed --- /dev/null +++ b/dendrite/logic/local/code/execute.py @@ -0,0 +1,26 @@ +from typing import Any + +from bs4 import BeautifulSoup +from .code_session import CodeSession + + +def execute(script: str, raw_html: str, return_data_json_schema) -> Any: + code_session = CodeSession() + soup = BeautifulSoup(raw_html, "lxml") + try: + + created_variables = code_session.exec_code(script, soup, raw_html) + + if "response_data" in created_variables: + response_data = created_variables["response_data"] + + try: + code_session.validate_response(return_data_json_schema, response_data) + except Exception as e: + raise Exception(f"Failed to validate response data. Exception: {e}") + + return response_data + else: + raise Exception("No return data available for this script.") + except Exception as e: + raise e diff --git a/dendrite/logic/local/dom/css.py b/dendrite/logic/local/dom/css.py new file mode 100644 index 0000000..a64833e --- /dev/null +++ b/dendrite/logic/local/dom/css.py @@ -0,0 +1,170 @@ +from typing import Optional +from bs4 import BeautifulSoup, Tag +from loguru import logger + + + +def find_css_selector(ele: Tag, soup: BeautifulSoup) -> str: + if ele is None: + return "" + + # Check for inherently unique elements + if ele.name in ["html", "head", "body"]: + return ele.name + + # List of attributes to check for unique selectors + priority_attrs = [ + "id", + "name", + "data-testid", + "data-cy", + "data-qa", + "aria-label", + "aria-labelledby", + "for", + "href", + "alt", + "title", + "role", + "placeholder", + ] + + # Try attrs + for attr in priority_attrs: + if attr_selector := check_unique_attribute(ele, soup, attr, ele.name): + return attr_selector + + # Try class combinations + if class_selector := find_unique_class_combination(ele, soup): + return class_selector + + # If still not unique, use parent selector with nth-child + parent_selector = find_selector_with_parent(ele, soup) + + return parent_selector + + +def check_unique_attribute( + ele: Tag, soup: BeautifulSoup, attr: str, tag_name: str +) -> str: + attr_value = ele.get(attr) + if attr_value: + attr_value = css_escape(attr_value) + attr = css_escape(attr) + selector = f'{css_escape(tag_name)}[{attr}="{attr_value}"]' + if check_if_selector_successful(selector, soup, True): + return selector + return "" + + +def find_unique_class_combination(ele: Tag, soup: BeautifulSoup) -> str: + classes = ele.get("class", []) + + if isinstance(classes, str): + classes = [classes] + + if not classes: + return "" + + tag_name = css_escape(ele.name) + + # Try single classes first + for cls in classes: + selector = f"{tag_name}.{css_escape(cls)}" + if check_if_selector_successful(selector, soup, True): + return selector + + # If single classes don't work, try the full combination + full_selector = f"{tag_name}{'.'.join([''] + [css_escape(c) for c in classes])}" + if check_if_selector_successful(full_selector, soup, True): + return full_selector + + return "" + + +def find_selector_with_parent(ele: Tag, soup: BeautifulSoup) -> str: + parent = ele.find_parent() + if parent is None or parent == soup: + return f"{css_escape(ele.name)}" + + parent_selector = find_css_selector(parent, soup) + siblings_of_same_type = parent.find_all(ele.name, recursive=False) + + if len(siblings_of_same_type) == 1: + return f"{parent_selector} > {css_escape(ele.name)}" + else: + index = position_in_node_list(ele, parent) + return f"{parent_selector} > {css_escape(ele.name)}:nth-child({index})" + + +def position_in_node_list(element: Tag, parent: Tag): + for index, child in enumerate(parent.find_all(recursive=False)): + if child == element: + return index + 1 + return -1 + +# https://github.com/mathiasbynens/CSS.escape +def css_escape(value): + if len(str(value)) == 0: + raise TypeError("`CSS.escape` requires an argument.") + + string = str(value) + length = len(string) + result = "" + first_code_unit = ord(string[0]) if length > 0 else None + + if length == 1 and first_code_unit == 0x002D: + return "\\" + string + + for index in range(length): + code_unit = ord(string[index]) + + if code_unit == 0x0000: + result += "\uFFFD" + continue + + if ( + (0x0001 <= code_unit <= 0x001F) + or code_unit == 0x007F + or (index == 0 and 0x0030 <= code_unit <= 0x0039) + or ( + index == 1 + and 0x0030 <= code_unit <= 0x0039 + and first_code_unit == 0x002D + ) + ): + result += "\\" + format(code_unit, "x") + " " + continue + + if ( + code_unit >= 0x0080 + or code_unit == 0x002D + or code_unit == 0x005F + or 0x0030 <= code_unit <= 0x0039 + or 0x0041 <= code_unit <= 0x005A + or 0x0061 <= code_unit <= 0x007A + ): + result += string[index] + continue + + result += "\\" + string[index] + + return result + +def check_if_selector_successful( + selector: str, bs4: BeautifulSoup, only_one: bool, +) -> Optional[str]: + + els = None + try: + els = bs4.select(selector) + except Exception as e: + logger.warning(f"Error selecting {selector}: {e}") + + if els: + if only_one and len(els) == 1: + return selector + elif not only_one and len(els) >= 1: + return selector + + return None \ No newline at end of file diff --git a/dendrite/browser/async_api/_dom/util/mild_strip.py b/dendrite/logic/local/dom/strip.py similarity index 100% rename from dendrite/browser/async_api/_dom/util/mild_strip.py rename to dendrite/logic/local/dom/strip.py diff --git a/dendrite/logic/local/dom/truncate.py b/dendrite/logic/local/dom/truncate.py new file mode 100644 index 0000000..6e09fc2 --- /dev/null +++ b/dendrite/logic/local/dom/truncate.py @@ -0,0 +1,72 @@ +import re + +def truncate_long_string( + val: str, + max_len_start: int = 150, + max_len_end: int = 150, + trucate_desc: str = "chars truncated for readability", +): + return ( + val + if len(val) < max_len_start + max_len_end + else val[:max_len_start] + + f"... [{len(val)-max_len_start-max_len_end} {trucate_desc}] ..." + + val[-max_len_end:] + ) + + +def truncate_long_string_w_words( + val: str, + max_len_start: int = 150, + max_len_end: int = 150, + trucate_desc: str = "words truncated for readability", + show_more_words_for_longer_val: bool = True, +): + if len(val) < max_len_start + max_len_end: + return val + else: + if show_more_words_for_longer_val: + max_len_end += int(len(val) / 100) + max_len_end += int(len(val) / 100) + + truncate_start_pos = max_len_start + steps_taken_start = 0 + while ( + truncate_start_pos > 0 + and val[truncate_start_pos] not in [" ", "\n"] + and steps_taken_start < 20 + ): + truncate_start_pos -= 1 + steps_taken_start += 1 + + truncate_end_pos = len(val) - max_len_end + steps_taken_end = 0 + while ( + truncate_end_pos < len(val) + and val[truncate_end_pos] not in [" ", "\n"] + and steps_taken_end < 20 + ): + truncate_end_pos += 1 + steps_taken_end += 1 + + if steps_taken_start >= 20 or steps_taken_end >= 20: + # Return simple truncation if we've looped further than 20 chars + return truncate_long_string(val, max_len_start, max_len_end, trucate_desc) + else: + return ( + val[:truncate_start_pos] + + f" [...{len(val[truncate_start_pos:truncate_end_pos].split())} {trucate_desc}...] " + + val[truncate_end_pos:] + ) + + +def remove_excessive_whitespace(text: str, max_whitespaces=1): + return re.sub(r"\s{2,}", " " * max_whitespaces, text) + + +def truncate_and_remove_whitespace(text, max_len_start=100, max_len_end=100): + return truncate_long_string_w_words( + remove_excessive_whitespace(text), + max_len_start=max_len_start, + max_len_end=max_len_end, + ) diff --git a/dendrite/logic/local/extract/extract_agent.py b/dendrite/logic/local/extract/extract_agent.py new file mode 100644 index 0000000..28e87e7 --- /dev/null +++ b/dendrite/logic/local/extract/extract_agent.py @@ -0,0 +1,529 @@ +import json +import re +from typing import Any, Optional +from anthropic.types import TextBlock +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.logic.local.dom.strip import mild_strip +from dendrite.logic.local.extract.prompts import create_script_prompt_segmented_html +from dendrite.logic.local.extract.scroll_agent import ScrollAgent +from dendrite.logic.local.get_element.hanifi_search import get_expanded_dom +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.page_information import PageInformation + +from bs4 import BeautifulSoup, Tag + +from dendrite.models.response.extract_page_response import ExtractPageResponse +from ..code.code_session import CodeSession +from ..ask.image import segment_image +from dendrite_server_merge.core.llm.claude import async_claude_request + +from dendrite_server_merge.logging import agent_logger + +from loguru import logger + + + +class ExtractAgent: + def __init__( + self, page_information: PageInformation, api_config: APIConfig, user_id: str + ) -> None: + self.page_information = page_information + self.soup = BeautifulSoup(page_information.raw_html, "lxml") + self.api_config = api_config + self.messages = [] + self.generated_script: Optional[str] = None + self.user_id = user_id + self.scroll_agent = ScrollAgent(api_config, page_information) + + def get_generated_script(self): + return self.generated_script + + async def write_and_run_script( + self, extract_page_dto: ExtractDTO + ) -> ExtractPageResponse: + mild_soup = mild_strip(self.soup) + + search_terms = [] + + segments = segment_image( + extract_page_dto.page_information.screenshot_base64, segment_height=4000 + ) + + scroll_result = await self.scroll_agent.scroll_through_page( + extract_page_dto.combined_prompt, + image_segments=segments, + ) + + if scroll_result.status == "error": + return ExtractPageResponse( + status="impossible", + message=str(scroll_result.message), + return_data=None, + used_cache=False, + created_script=None, + ) + + if scroll_result.status == "loading": + return ExtractPageResponse( + status="loading", + message="This page is still loading. Please wait a bit longer.", + return_data=None, + used_cache=False, + created_script=None, + ) + + expanded_html = None + if scroll_result.element_to_inspect_html: + combined_prompt = ( + "Get these elements (make sure you only return element that you are confident that these are the correct elements, it's OK to not select any elements):\n- " + + "\n- ".join(scroll_result.element_to_inspect_html) + ) + expanded = await get_expanded_dom( + mild_soup, + combined_prompt, + self.api_config, + ) + if expanded: + expanded_html = expanded[0] + + if expanded_html: + return await self.code_script_from_found_expanded_html_tags( + extract_page_dto, expanded_html, segments + ) + else: + compress_html = CompressHTML( + mild_soup, + exclude_dendrite_ids=False, + focus_on_text=True, + max_token_size=16000, + max_size_per_element=10000, + compression_multiplier=0.5, + ) + expanded_html = await compress_html.compress(search_terms) + return await self.code_script_from_compressed_html( + extract_page_dto, expanded_html, segments, mild_soup + ) + + def segment_large_tag(self, tag): + segments = [] + current_segment = "" + current_tokens = 0 + for line in tag.split("\n"): + line_tokens = token_count(line) + if current_tokens + line_tokens > 4000: + segments.append(current_segment) + current_segment = line + current_tokens = line_tokens + else: + current_segment += line + "\n" + current_tokens += line_tokens + if current_segment: + segments.append(current_segment) + return segments + + async def code_script_from_found_expanded_html_tags( + self, extract_page_dto: ExtractDTO, expanded_html, segments + ): + agent_logger.info("Starting code_script_from_found_expanded_html_tags method") + messages = [] + + user_prompt = create_script_prompt_segmented_html( + extract_page_dto.combined_prompt, + expanded_html, + self.page_information.url, + ) + agent_logger.debug(f"User prompt created: {user_prompt[:100]}...") + + content = [ + { + "type": "text", + "text": user_prompt, + }, + ] + + messages = [ + {"role": "user", "content": content}, + ] + + iterations = 0 + max_retries = 10 + + generated_script: str = "" + response_data: Any | None = None + + while iterations <= max_retries: + iterations += 1 + agent_logger.info(f"Starting iteration {iterations}") + + config = { + "messages": messages, + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.3, + "max_tokens": 1500, + } + res = await async_claude_request(config, self.api_config) + if not isinstance(res.content[0], TextBlock): + logger.error("Needs to be an text block: ", res) + raise Exception("Needs to be an text block") + + text = res.content[0].text + dict_res = { + "role": "assistant", + "content": text, + } + messages.append(dict_res) + + json_pattern = r"```json(.*?)```" + code_pattern = r"```python(.*?)```" + + if text: + json_matches = re.findall(json_pattern, text, re.DOTALL) + code_matches = re.findall(code_pattern, text, re.DOTALL) + + if len(json_matches) + len(code_matches) > 1: + content = "Error: Please output only one action at a time (either JSON or Python code, not both)." + messages.append({"role": "user", "content": content}) + continue + + for code_match in code_matches: + agent_logger.debug("Processing code match") + generated_script = code_match.strip() + temp_code_session = CodeSession() + try: + variables = temp_code_session.exec_code( + generated_script, + self.soup, + self.page_information.raw_html, + ) + agent_logger.debug("Code execution successful") + except Exception as e: + agent_logger.error(f"Code execution failed: {str(e)}") + content = f"Error: {str(e)}" + messages.append({"role": "user", "content": content}) + continue + + try: + if "response_data" in variables: + response_data = variables["response_data"] + # agent_logger.debug(f"Response data: {response_data}") + + if extract_page_dto.return_data_json_schema != None: + temp_code_session.validate_response( + extract_page_dto.return_data_json_schema, + response_data, + ) + + llm_readable_exec_res = ( + temp_code_session.llm_readable_exec_res( + variables, + extract_page_dto.combined_prompt, + iterations, + max_retries, + ) + ) + + messages.append( + {"role": "user", "content": llm_readable_exec_res} + ) + continue + else: + content = ( + f"Error: You need to add the variable 'response_data'" + ) + messages.append( + { + "role": "user", + "content": content, + } + ) + continue + except Exception as e: + llm_readable_exec_res = temp_code_session.llm_readable_exec_res( + variables, + extract_page_dto.combined_prompt, + iterations, + max_retries, + ) + content = f"Error: Failed to validate `response_data`. Exception: {e}. {llm_readable_exec_res}" + messages.append( + { + "role": "user", + "content": content, + } + ) + continue + + for json_match in json_matches: + agent_logger.debug("Processing JSON match") + extracted_json = json_match.strip() + data_dict = json.loads(extracted_json) + current_segment = 0 + if "request_more_html" in data_dict: + agent_logger.info("Processing element indexes") + try: + current_segment += 1 + content = f"""Here is more of the HTML:\n```html\n{expanded_html[LARGE_HTML_CHAR_TRUNCATE_LEN*current_segment:LARGE_HTML_CHAR_TRUNCATE_LEN*(current_segment+1)]}\n```""" + if len(expanded_html) > LARGE_HTML_CHAR_TRUNCATE_LEN * ( + current_segment + 1 + ): + content += "\nThere is still more HTML to see. You can request more if needed." + else: + content += "\nThis is the end of the HTML content." + messages.append({"role": "user", "content": content}) + continue + except Exception as e: + agent_logger.error( + f"Error processing element indexes: {str(e)}" + ) + content = f"Error: {str(e)}" + messages.append({"role": "user", "content": content}) + continue + elif "error" in data_dict: + agent_logger.error(f"Error in data_dict: {data_dict['error']}") + raise HTTPException(404, detail=data_dict["error"]) + elif "success" in data_dict: + agent_logger.info("Script generation successful") + self.generated_script = generated_script + + await upsert_script_in_db( + extract_page_dto.combined_prompt, + generated_script, + extract_page_dto.page_information.url, + user_id=self.user_id, + ) + # agent_logger.debug(f"Response data: {response_data}") + return ExtractPageResponse( + status="success", + message=data_dict["success"], + return_data=response_data, + used_cache=False, + created_script=self.get_generated_script(), + ) + + agent_logger.warning("Failed to create script after retrying several times") + return ExtractPageResponse( + status="failed", + message="Failed to create script after retrying several times.", + return_data=None, + used_cache=False, + created_script=self.get_generated_script(), + ) + + async def code_script_from_compressed_html( + self, extract_page_dto: ExtractDTO, expanded_html, segments, mild_soup + ): + messages = [] + + user_prompt = create_script_prompt_compressed_html( + extract_page_dto.combined_prompt, + expanded_html, + self.page_information.url, + ) + + content = [ + { + "type": "text", + "text": user_prompt, + }, + ] + + if extract_page_dto.use_screenshot: + content += [ + { + "type": "text", + "text": "Here is a screenshot of the website:", + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": segments[0], + }, + }, + ] + + messages = [ + {"role": "user", "content": content}, + ] + + iterations = 0 + max_retries = 10 + + generated_script: str = "" + response_data: Any | None = None + + while iterations <= max_retries: + iterations += 1 + + config = { + "messages": messages, + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.3, + "max_tokens": 1500, + } + res = await async_claude_request(config, self.api_config) + if not isinstance(res.content[0], TextBlock): + logger.error("Needs to be an text block: ", res) + raise Exception("Needs to be an text block") + + text = res.content[0].text + dict_res = { + "role": "assistant", + "content": text, + } + messages.append(dict_res) + + json_pattern = r"```json(.*?)```" + code_pattern = r"```python(.*?)```" + + if text: + json_matches = re.findall(json_pattern, text, re.DOTALL) + code_matches = re.findall(code_pattern, text, re.DOTALL) + + if len(json_matches) + len(code_matches) > 1: + content = "Error: Please output only one action at a time (either JSON or Python code, not both)." + messages.append({"role": "user", "content": content}) + continue + + for code_match in code_matches: + generated_script = code_match.strip() + temp_code_session = CodeSession() + try: + variables = temp_code_session.exec_code( + generated_script, + self.soup, + self.page_information.raw_html, + ) + except Exception as e: + content = f"Error: {str(e)}" + messages.append({"role": "user", "content": content}) + continue + + try: + if "response_data" in variables: + response_data = variables["response_data"] + # agent_logger.debug(f"Response data: {response_data}") + + if extract_page_dto.return_data_json_schema != None: + temp_code_session.validate_response( + extract_page_dto.return_data_json_schema, + response_data, + ) + + llm_readable_exec_res = ( + temp_code_session.llm_readable_exec_res( + variables, + extract_page_dto.combined_prompt, + iterations, + max_retries, + ) + ) + + messages.append( + {"role": "user", "content": llm_readable_exec_res} + ) + continue + else: + content = ( + f"Error: You need to add the variable 'response_data'" + ) + messages.append( + { + "role": "user", + "content": content, + } + ) + continue + except Exception as e: + llm_readable_exec_res = temp_code_session.llm_readable_exec_res( + variables, + extract_page_dto.combined_prompt, + iterations, + max_retries, + ) + content = f"Error: Failed to validate `response_data`. Exception: {e}. {llm_readable_exec_res}" + messages.append( + { + "role": "user", + "content": content, + } + ) + continue + + for json_match in json_matches: + extracted_json = json_match.strip() + data_dict = json.loads(extracted_json) + content = "" + if "d-ids" in data_dict: + try: + + content += "Here is the expanded HTML:" + for d_id in data_dict["d-ids"]: + d_id_res = mild_soup.find(attrs={"d-id": d_id}) + + tag = None + + if isinstance(d_id_res, Tag): + tag = d_id_res + + if tag: + subsection_mild = mild_strip(tag) + pretty = subsection_mild.prettify() + if len(pretty) > 120000: + compress_html = CompressHTML( + subsection_mild, + exclude_dendrite_ids=False, + max_token_size=16000, + max_size_per_element=10000, + focus_on_text=True, + compression_multiplier=0.3, + ) + subsection_compressed_html = ( + await compress_html.compress() + ) + content += f"\n\nThis expanded element with the d-id '{d_id}' was too large to inspect fully! Here is a compressed version of the element you selected, please inspect a smaller section of it:\n```html\n{subsection_compressed_html}\n```" + else: + subsection_mild = mild_strip( + tag, keep_d_id=False + ) + pretty = subsection_mild.prettify() + content += f"\n\nExpanded element with the d-id '{d_id}':\n```html\n{pretty}\n```" + else: + content += f"\n\nNo valid element could be found with the d-id or id '{d_id}'. Prefer using the d-id attribute." + + content += "\n\nIf you cannot find the relevant data in this HTML, consider expanding a different region." + messages.append({"role": "user", "content": content}) + continue + except Exception as e: + messages.append( + {"role": "user", "content": f"Error: {str(e)}"} + ) + agent_logger.debug(f"role: user, content: Error: {str(e)}") + elif "error" in data_dict: + raise HTTPException(404, detail=data_dict["error"]) + elif "success" in data_dict: + self.generated_script = generated_script + + await upsert_script_in_db( + extract_page_dto.combined_prompt, + generated_script, + extract_page_dto.page_information.url, + user_id=self.user_id, + ) + + return ExtractPageResponse( + status="success", + message=data_dict["success"], + return_data=response_data, + used_cache=False, + created_script=self.get_generated_script(), + ) + + return ExtractPageResponse( + status="failed", + message="Failed to create script after retrying several times.", + return_data=None, + used_cache=False, + created_script=self.get_generated_script(), + ) diff --git a/dendrite/logic/local/extract/prompts.py b/dendrite/logic/local/extract/prompts.py new file mode 100644 index 0000000..4171ce7 --- /dev/null +++ b/dendrite/logic/local/extract/prompts.py @@ -0,0 +1,230 @@ +def get_script_prompt(final_compressed_html: str, prompt: str, current_url: str): + return f"""Compressed HTML: +{final_compressed_html} + +Please look at the HTML DOM above and use execute_code to accomplish the user's task. + +Don't use the attributes 'is-compressed' and 'd-id' inside your script. + +Prefer using soup.select() over soup.find_all(). + +If you are asked to fetch text from an article or similar it's generally a good idea to find the element(s) containing the article text and extracting the text from those. You'll also need to remove unwanted text from elements that isn't article text. + +All elements with the attribute is-compressed="true" are collapsed and may contain hidden elements. If you need to use an element that is compressed you have to call expand_html_further, example: + +expand_html_further({{"prompt": "I need to understand the structure of at least one product to create a script that fetches each product, since all the products are compressed I'll expand the first two ones. I'll also expand the pagenation controls since they are relevant for the task.", "d-ids_to_expand": "3uy9v2, 3uy9d2, -29ahd"}}) + +When scraping a list of items make sure at least one of the items is fully expanded to understand each items' structure before you code. You don't need to expand all items if you can see that there is a repeating structure. + +You code must be a full implementation that solves the user's task. + +Try to make your scripts as general as possible. They should work for different pages with a similar html structure if possible. No hard-coded values that'll only work for the page above. + +Finally, the script must contain a variable called 'response_data'. This variable is sent back to the user and it must match the match the specification inside their prompt listed below. + +Current URL: {current_url} +User's Prompt: +{prompt}""" + + +def expand_futher_prompt( + compressed_html: str, + max_iterations: int, + iterations: int, + reasoning_prompt: str, + question: str, +): + return f"""{compressed_html} + +Please look at the compressed HTML above and output a comma separated of elements that need to be de-compressed so that the task can be solved. + +Task: '{question}' + +Every element with the attribute is-compressed="true" can be de-compressed. Compressed elements may contain hidden elements such as anchor tags and buttons, so it's really important that relevant element to the task are expanded. + +You'll get max {max_iterations} interations to explore the HTML DOM Tree. + +You are currently on iteration {iterations}. Try to expand the DOM in relevant places at least three times. + +{reasoning_prompt} + +It's really important that you expand ALL the elements you believe could be useful for the task! However, in situations where you have repeating elements, such as products elements in a product list or sections of paragraphs in an article, you only need to expand a few of the repeating elements to be able to understand the others' structure. + +Now you may output: +- Ids to inspect further prefixed by some short reasoning (Don't expand irrelevant element and avoid outputting many IDs since that increases the token size of the HTML preview) +- "Done" once every relevant element is expanded. +- An error message if the task is too vauge or not possible to complete. A common use-case for the error message is when a page loads incorrectly and none of the task's data is available. + +See the examples below to see each outputs format: + +EXAMPLE OUTPUT +Reasoning: Most of the important elements are expanded, but I still need to understand the article's headings' HTML structure. To do this I'll expand the first section heading with the text 'hello kitty' and the d-id adh2ia. I'll also expand the related infobox with the id -s29as. By expanding these I'll be able to understand all the article's titles. +Ids: adh2ia, -s29as +END EXAMPLE OUTPUT + +EXAMPLE OUTPUT +Reasoning: To understand the structure of the compressed product cards in the product list I'll expand the three first ones with the d-ids -7ap2j1, -7ap288 and -7ap2au. I'll also the pagenation controls at the bottom of the product list since pagenation can be useful for the task, this includes the page buttons for '1', '2' and '3' button with the d-ids j02ajd, j20had, j9dwh9 and the 'next page' button with the id j9dwss. +Ids: -7ap2j1, -7ap288, -7ap2au, j02ajd, j20had, j9dwh9, j9dwss +END EXAMPLE OUTPUT + +EXAMPLE OUTPUT +Done +END EXAMPLE OUTPUT + +EXAMPLE OUTPUT +Error: I don't understand what is mean with 'extract the page text', this page is completely empty. +END EXAMPLE OUTPUT""" + + +def create_script_prompt_compressed_html( + combined_prompt: str, + expanded_html: str, + current_url: str, +): + return f"""You are a web scraping agent that runs one action at a time by outputting a message with either elements to decompress, code to run or a status message. Never run several actions in the same message. + +Code a bs4 or regex script that can solve the task listed below for the webpage I'll specify below. First, inspect relevant areas of the DOM. + + +{combined_prompt} + + +Here is a compressed version of the webpage's HTML: + +```html +{expanded_html} +``` + + +Important: Every element with the attribute `is-compressed="true"` is compressed – compressed elements may contain hidden elements such as anchor tags and buttons, so you need to decompress them to fully understand their structure before you write a script! + +Below are your available functions and how to use them: + +Start by outputting one or more d-ids of elements you'd like to decompress before you right a script. Focus on decompressing elements that look relevant to the task. If possible, expand one d-id at a time. Output in a format like this: + +[Short reasoning first.] +```json +{{ + "d-ids": ["xxx", "yyy"] +}} +``` + +Once you have decompressed the DOM at least one time in separate messages and have a good enough understanding of the page's structure, write some python code to extract the required data using bs4 or regex. `from datetime import datetime` is available. + +Your code will be ran inside exec() so don't use a return statement, just create variables. + +To scrape information from the current page use the predefined variable `html_string` (all the page's html as a string) or `soup` (current page's root's bs4 object). Don't use 'd-id' and 'is_compressed' in your script since these are temporary. Use selectors native to the site. + +The script must contain a variable called 'response_data' and it's structure must match the task listed above. + +Don't return a response_data with hardcoded values that only work for the current page. The script must be general and work for similar pages with the same structure. + +Unless specified, return an exception if a expected value cannot be extracted. + +The current URL is: {current_url} + +Here's how you can do it in a message: + +[Do some reasoning first] +```python +# Simple bs4 code that fetches all the page's hrefs +response_data = [a.get('href') for a in soup.find_all('a')] # Uses the predefined soup variable +``` + +If the task isn't possible to complete (maybe because the task is too vauge, the page contains an error or the page failed to load) don't try and create a script with many assumptions. Instead, output an error like this: + +```json +{{ + "error": "error message" +}} +``` + +Once you've created and ran a script and you are happy with response_data, output a short success message (max one paragraph) containing json like this, the response_data will automatically be returned to the user once you send this message, you don't need to output it: + +```json +{{ + "success": "Write one-two sentences about how your the script works and how you ended up with the result you got." +}} +``` + +Don't include both the python code and json object in the same message. + +Be sure that the the script has been execucted and you have seen the response_data in a previous message before you output the success message.""" + + +LARGE_HTML_CHAR_TRUNCATE_LEN = 40000 + + +def create_script_prompt_segmented_html( + combined_prompt: str, + expanded_html: str, + current_url: str, +): + if len(expanded_html)/4 > LARGE_HTML_CHAR_TRUNCATE_LEN: + html_prompt = f"""```html + {expanded_html[:LARGE_HTML_CHAR_TRUNCATE_LEN]} +``` +This HTML is truncated to {LARGE_HTML_CHAR_TRUNCATE_LEN} characters since it was too large. If you need to see more of the HTML, output a message like this: +```json +{{ + "request_more_html": true +}} +``` +""" + else: + html_prompt = f"""```html + {expanded_html} +``` +""" + + return f"""You are a web scraping agent that analyzes HTML and writes Python scripts to extract data. Your task is to solve the following request for the webpage specified below. + + +{combined_prompt} + + +Current URL: {current_url} + +Here is a truncated version of the HTML that focuses on relevant parts of the webpage (some elements are have been replaced with their text contents): +{html_prompt} + +Instructions: +1. Analyze the provided HTML segments carefully. + +2. Use bs4 or regex. `from datetime import datetime` is available. +- Your code will be ran inside exec() so don't use a return statement, just create variables. +- To scrape information from the current page use the predefined variable `html_string` (all the page's html as a string) or `soup` (current page's root's bs4 object). Don't use 'd-id' and 'is_compressed' in your script since these are temporary. Use selectors native to the site. +- The script must contain a variable called 'response_data' and it's structure must match the task listed above. +- Don't return a response_data with hardcoded values that only work for the current page. The script must be general and work for similar pages with the same structure. +- Unless specified, return an exception if a expected value cannot be extracted. + +3. Output your Python script in this format: +[Do some reasoning first] +```python +# Simple bs4 code that fetches all the page's hrefs +response_data = [a.get('href') for a in soup.find_all('a')] # Uses the predefined soup variable +``` + +Don't output an explaination of the script after the code. Just do some short reasoning before. + +4. If the task isn't possible to complete, output an error message like this: +```json +{{ + "error": "Detailed error message explaining why the task can't be completed" +}} +``` + +5. Once you've successfully created and ran a script, seen that the output is correct and you're happy with it, output a short success message: +```json +{{ + "success": "Brief explanation of how your script works and how you arrived at the result" +}} +``` +Remember: +- Only output one action at a time (element index to expand, Python code, or status message). +- Don't include both Python code and JSON objects in the same message. +- Ensure the script has been executed and you've seen the `response_data` before sending the success message. +- Do short reasoning before you output an action, max one-two sentences. +- Never include a success message in the same output as your Python code. Always output the success message after you've seen the result of your code. + +You may now begin by analyzing the HTML or requesting to expand specific elements if needed.""" diff --git a/dendrite/logic/local/extract/scroll_agent.py b/dendrite/logic/local/extract/scroll_agent.py new file mode 100644 index 0000000..aa40ed8 --- /dev/null +++ b/dendrite/logic/local/extract/scroll_agent.py @@ -0,0 +1,246 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +import json +import re +from typing import List, Literal, Optional + +from anthropic.types import TextBlock +from loguru import logger + +from dendrite.models.page_information import PageInformation +from dendrite_server_merge.core.llm.claude import async_claude_request + + +ScrollActionStatus = Literal["done", "scroll_down", "loading", "error"] + + +@dataclass +class ScrollResult: + element_to_inspect_html: List[str] + segment_index: int + status: ScrollActionStatus + message: Optional[str] = None + + +class ScrollRes(ABC): + @abstractmethod + def parse(self, data_dict: dict, segment_i: int) -> Optional[ScrollResult]: + pass + + +class ElementPromptsAction(ScrollRes): + def parse(self, data_dict: dict, segment_i: int) -> Optional[ScrollResult]: + if "element_to_inspect_html" in data_dict: + + status = ( + "scroll_down" + if not data_dict.get("continue_scrolling", False) + else "done" + ) + + return ScrollResult(data_dict["element_to_inspect_html"], segment_i, status) + return None + + +class LoadingAction(ScrollRes): + def parse(self, data_dict: dict, segment_i: int) -> Optional[ScrollResult]: + if data_dict.get("is_loading", False): + return ScrollResult([], segment_i, "loading") + return None + + +class ErrorRes(ScrollRes): + def parse(self, data_dict: dict, segment_i: int) -> Optional[ScrollResult]: + if "error" in data_dict: + return ScrollResult( + [], + segment_i, + "error", + data_dict["error"], + ) + return None + + +class ScrollAgent: + def __init__(self, api_config, page_information: PageInformation): + self.api_config = api_config + self.page_information = page_information + self.choices: List[ScrollRes] = [ + ElementPromptsAction(), + LoadingAction(), + ErrorRes(), + ] + + async def scroll_through_page( + self, + combined_prompt: str, + image_segments: List[str], + ) -> ScrollResult: + messages = self.create_initial_message(combined_prompt, image_segments[0]) + all_elements_to_inspect_html = [] + current_segment = 0 + + while current_segment < len(image_segments): + data_dict = await self.process_segment(messages) + + for choice in self.choices: + result = choice.parse(data_dict, current_segment) + if result: + if result.element_to_inspect_html: + all_elements_to_inspect_html.extend( + result.element_to_inspect_html + ) + return result + + if "element_to_inspect_html" in data_dict: + all_elements_to_inspect_html.extend( + data_dict["element_to_inspect_html"] + ) + + if self.should_continue_scrolling( + data_dict, current_segment, len(image_segments) + ): + current_segment += 1 + scroll_message = self.create_scroll_message( + image_segments[current_segment] + ) + messages.append(scroll_message) + else: + break + + return ScrollResult(all_elements_to_inspect_html, current_segment, "done") + + async def process_segment(self, messages: List[dict]) -> dict: + config = { + "messages": messages, + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.3, + "max_tokens": 1500, + } + res = await async_claude_request(config, self.api_config) + + if not isinstance(res.content[0], TextBlock): + raise Exception("Response needs to be a text block") + + text = res.content[0].text + messages.append({"role": "assistant", "content": text}) + + json_pattern = r"```json(.*?)```" + + json_matches = re.findall(json_pattern, text, re.DOTALL) + + if len(json_matches) > 1: + logger.warning("Agent output multiple actions in one message") + error_message = "Error: Please output only one action at a time." + messages.append({"role": "user", "content": error_message}) + elif json_matches: + return json.loads(json_matches[0].strip()) + + error_message = "No valid JSON found in the response" + logger.error(error_message) + messages.append({"role": "user", "content": error_message}) + raise Exception(error_message) + + def create_initial_message( + self, combined_prompt: str, first_image: str + ) -> List[dict]: + content = [ + { + "type": "text", + "text": f"""You are a web scraping agent that can code scripts to solve the web scraping tasks listed below for the webpage I'll specify. Before we start coding, we need to inspect the html of the page closer. + +This is the web scraping task: + +{combined_prompt} + + +Analyze the viewport and decide on the next action: + +1. Identify elements that we want to inspect closer so we can write the script. Do this by outputting a message with a list of prompts to find the relevant element(s). + +Output as few elements as possible, but it should be enought to gain a proper understanding of the DOM for our script. + +If a list of items need to be extracted, consider getting a few unique examples of items from the list that differ slightly so we can create code that accounts for their differences. Avoid listing several elements that are very similar since we can infer the structure of one or two of them to the rest. + +Don't get several different parts of one relevant element, just get the whole element since it's easier to just inspect the whole element. + +Avoid selecting very large elements that contain a lot of html since it can be very overwhelming to inspect. + +Always be specific about the element you are thinking of, don't write 'get a item', write 'get the item with the text "Item Name"'. + +Here's an example of a good output: +[Short reasoning first, max one paragraph] +```json +{{ + "element_to_inspect_html": ["The small container containing the weekly amount of downloads, labeled 'Weekly Downloads'", "The element containing the main body of article text, the title is 'React Router DOM'."], + "continue_scrolling": true/false (only scroll down if you think more relevant elements are further down the page, only do this if you need to) +}} +``` + +2. If you can't see relevant elements just yet, but you think more data might be available further down the page, output: +[Short reasoning first, max one paragraph] +```json +{{ + "scroll_down": true +}} +``` + +3. This page was first loaded {round(self.page_information.time_since_frame_navigated, 2)} second(s) ago. If the page is blank or the data is not available on the current page it could be because the page is still loading. If you believe this is the case, output: +[Short reasoning first, max one paragraph] +```json +{{ + "is_loading": true +}} +``` + +4. In case you the data is not available on the current page and the task does not describe how to handle the non-available data, or there seems to be some kind of mistake, output a json with a short error message, like this: +[Short reasoning first, max one paragraph] +```json +{{ + "error": "This page doesn't contain any package data, welcome page for 'dendrite.systems', it won't be possible to code a script to extract the requested data.", + "was_blocked_by_recaptcha": true/false +}} +``` + +Continue scrolling and accumulating element prompts until you feel like we have enough elements to inspect to create an excellent script. + +Important: Only output one json object per message. + +Below is a screenshot of the current page, if it looks blank or empty it could still be loading. If this is the case, don't guess what elements to inspect, respond with is loading.""", + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": first_image, + }, + }, + ] + + return [ + {"role": "user", "content": content}, + ] + + def create_scroll_message(self, image: str) -> dict: + return { + "role": "user", + "content": [ + {"type": "text", "text": "Scrolled down, here is the new viewport:"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": image, + }, + }, + ], + } + + def should_continue_scrolling( + self, data_dict: dict, current_index: int, total_segments: int + ) -> bool: + return ( + "scroll_down" in data_dict or data_dict.get("continue_scrolling", False) + ) and current_index + 1 < total_segments diff --git a/dendrite/logic/local/get_element/agents/agent.py b/dendrite/logic/local/get_element/agents/agent.py new file mode 100644 index 0000000..cfa1e0b --- /dev/null +++ b/dendrite/logic/local/get_element/agents/agent.py @@ -0,0 +1,146 @@ +from abc import ABC, abstractmethod +import json +from typing import Any, Dict, Generic, List, Literal, Optional, Type, TypeVar + +from anthropic.types import Message, TextBlock + +from dendrite_server_merge.core.llm.claude import async_claude_request +from dendrite_server_merge.core.llm.gemini import async_gemini_request +from dendrite_server_merge.core.llm.openai import async_openai_request +from dendrite_server_merge.models.APIConfig import APIConfig + + +T = TypeVar("T") +U = TypeVar("U") + + +class Agent(ABC, Generic[T, U]): + def __init__( + self, + model: U, + api_config: APIConfig, + system_message: Optional[str] = None, + temperature: float = 0, + max_tokens: int = 1500, + ): + self.messages: List[Dict] = [] + self.api_config = api_config + self.model = model + self.temperature = temperature + self.max_tokens = max_tokens + + if system_message: + self._add_system_message(system_message) + + @abstractmethod + def _add_system_message(self, message: str) -> None: + pass + + @abstractmethod + async def add_message(self, message: str) -> T: + pass + + +class AnthropicAgent( + Agent[Message, Literal["claude-3-5-sonnet-20241022", "claude-3-haiku-20240307"]] +): + def __init__( + self, + model: Literal["claude-3-5-sonnet-20241022", "claude-3-haiku-20240307"], + api_config: APIConfig, + system_message: Optional[str] = None, + temperature: float = 0, + max_tokens: int = 1500, + enable_caching: bool = False, # Add enable_caching option + ): + self.enable_caching = enable_caching # Store the caching preference + super().__init__(model, api_config, system_message, temperature, max_tokens) + + def _add_system_message(self, message: str) -> None: + self.system_msg: List[dict] = [{"type": "text", "text": message}] + if self.enable_caching: + self.system_msg[0]["cache_control"] = {"type": "ephemeral"} + + async def add_message(self, message: str) -> Message: + self.messages.append({"role": "user", "content": message}) + + spec = { + "messages": self.messages, + "model": self.model, + "temperature": self.temperature, + "max_tokens": self.max_tokens, + } + + if self.system_msg: + spec["system"] = self.system_msg + + res = await async_claude_request( + spec, self.api_config, enable_caching=self.enable_caching + ) + + if isinstance(res.content[0], TextBlock): + self.messages.append({"role": "assistant", "content": res.content[0].text}) + else: + raise ValueError("Unexpected response type: ", type(res.content[0])) + + return res + + def dump_messages(self) -> str: + return json.dumps( + [{"role": "system", "content": self.system_msg}] + self.messages, indent=2 + ) + + +class OpenAIAgent( + Agent[ChatCompletion, Literal["gpt-3.5", "gpt-4", "gpt-4o", "gpt-4o-mini"]] +): + + def _add_system_message(self, message: str): + self.messages.append({"role": "system", "content": message}) + + async def add_message(self, message: str) -> ChatCompletion: + self.messages.append({"role": "user", "content": message}) + + spec = { + "messages": self.messages, + "model": self.model, + "temperature": self.temperature, + "max_tokens": self.max_tokens, + } + + res = await async_openai_request(spec, self.api_config) + self.messages.append( + {"role": "assistant", "content": res.choices[0].message.content} + ) + return res + + def dump_messages(self) -> str: + return json.dumps(self.messages, indent=2) + + +class GoogleAgent(Agent[AsyncGenerateContentResponse, str]): + + def _add_system_message(self, message: str): + self.system_message = message + + async def add_message(self, message: str) -> AsyncGenerateContentResponse: + self.messages.append({"role": "user", "parts": message}) + + res = await async_gemini_request( + system_message=self.system_message, + llm_config_dto=self.api_config, + model_name="gemini-1.5-flash", + contents=self.messages, + ) + + res.candidates[0].content.parts[0] + self.messages.append( + {"role": "assistant", "parts": res.candidates[0].content.parts} + ) + + return res + + def dump_messages(self) -> str: + return json.dumps( + [{"role": "system", "parts": "system_message"}] + self.messages, indent=2 + ) \ No newline at end of file diff --git a/dendrite/logic/local/get_element/agents/prompts/__init__.py b/dendrite/logic/local/get_element/agents/prompts/__init__.py new file mode 100644 index 0000000..b91825e --- /dev/null +++ b/dendrite/logic/local/get_element/agents/prompts/__init__.py @@ -0,0 +1,12 @@ +def load_prompt(prompt_path: str) -> str: + with open(prompt_path, "r") as f: + prompt = f.read() + return prompt + + +SEGMENT_PROMPT = load_prompt( + "dendrite_server_merge/core/web_scraping_agent/get_interactable/agents/prompts/segment.prompt" +) +SELECT_PROMPT = load_prompt( + "dendrite_server_merge/core/web_scraping_agent/get_interactable/agents/prompts/select.prompt" +) diff --git a/dendrite/logic/local/get_element/agents/prompts/segment.prompt b/dendrite/logic/local/get_element/agents/prompts/segment.prompt new file mode 100644 index 0000000..eb84be1 --- /dev/null +++ b/dendrite/logic/local/get_element/agents/prompts/segment.prompt @@ -0,0 +1,107 @@ +You are an agent that is given the task to find candidate elements that match the element that the user is looking for. You will get multiple segments of the html of the page and a description of the element that the user is looking for. +The description can be the text that the element contains, the type of element. You might get both short and long descriptions. +Don't only look for the exact match of the text. + +Look at aria-label if there are any as they are helpful in identifying the elements. + +You will get the information in the following format: + + + DESCRIPTION + + + + HTML CONTENT + +... + + HTML CONTENT + + +Each element will have an attribute called d-id which you should refer to if you can find the elements that the user is looking for. There might be multiple elements that are fit the user's request, if so include multiple d_id:s. +If you've selected an element you should NOT select another element that is a child of the element you've selected. +Be sure to include a reason for why you selected the elements that you did. Think step by step, what made you choose this element over the others. +Your response should include 2-3 sentences of reasoning and a code block containing json including the backticks, the reason text is just a placeholder. Always include a sentence of reasoning in the output: + +```json +{ + "reason": , + "d_id": ["125292", "9541ad"], + "status": "success" +} +``` + +If no element seems to match the user's request, or you think the page is still loading, output the following with 2-3 sentences of reasoning in the output: + +```json +{ + "reason": , + "status": "failed" or "loading" +} +``` + +Here are some examples to help you understand the task (your response is the content under "Assistant:"): + +Example 1: + +USER: Can you get the d_id of the element that matches this description? + + + pull requests count + + + +
  • + + + Pull requests + + + 14 + + +
    • +     + +ASSISTANT: + +```json +{ + "reason": "I selected this element because it has the class Counter and is a number next to the pull requests text.", + "d_id": ["235512"], + "status": "success" +} +``` + +Example 2: + +USER: Can you get the d_id of the element that matches this description? + + + search bar + + + +
    or tags or their content in your response. \ No newline at end of file diff --git a/dendrite/logic/local/get_element/agents/prompts/select.prompt b/dendrite/logic/local/get_element/agents/prompts/select.prompt new file mode 100644 index 0000000..339d328 --- /dev/null +++ b/dendrite/logic/local/get_element/agents/prompts/select.prompt @@ -0,0 +1,90 @@ +You are a web scraping agent who is an expert at selecting element(s) that the user is asking for. + +You will get the information in the following format: + + + DESCRIPTION + + +```html +HTML CONTENT +``` + +Try to select a single element that you think is the best match for the user's request. The element should be as small as possible while still containing the information that the user is looking for. If there are wrappers select the element inside. Be sure to include a reason for why you selected the element that you did. +To select an element you should refer to the d-id attribute which is a unique identifier for each element. + +Your response should be in the following format, including the backticks. Do all your reasoning in the `reason` field, only output the json: + +```json +{ + "reason": "After looking at the HTML it is clear that '98jorq3' is the correct element since is contains the text 'Hello World' which is exactly what the user asked for.", + "d_ids": ["98jorq3"], + "status": "success" +} +``` + +If the requested element doesn't seem to be available on the page, that's OK. Return the following format, including the backticks: + +```json +{ + "reason": "This page doesn't seem to contain any link for a 'Github repository' as requested. The page has had a couple of seconds to load too and there are links for twitter and facebook, but no github. So, it's impossible to find the requested element on this page.", + "status": "impossible" +} +``` + +A page could still be loading, if this is the case you should return the following format, including the backticks: + +```json +{ + "reason": "Since the requested element is missing and the page only loaded in 2 seconds ago, I believe the page is still loading. Let's wait for the page to load and try again.", + "status": "loading" +} +``` + +Here is an example to help you understand how to select the best element: + +USER: + + pull requests count next to commits count + + +```html + + + ... +
  • + + + Commits + + + 24 + + +
    • +
  • + + + Pull requests + + + 14 + + +
    • + ... + + +``` + +ASSISTANT: +```json +{ + "reason": "This is tricky, there are a few elements that could match the user's request (s8yy81 and 781faa), however I selected the element with the d-id 's8yy81' because the span a class Counter and contains a number and is next to a span with the text pull requests.", + "d_ids": ["s8yy81"], + "status": "success" +} +``` + +IMPORTANT! +Your reasoning must be limited to 3-4 sentences. diff --git a/dendrite/logic/local/get_element/agents/segment_agent.py b/dendrite/logic/local/get_element/agents/segment_agent.py new file mode 100644 index 0000000..1251582 --- /dev/null +++ b/dendrite/logic/local/get_element/agents/segment_agent.py @@ -0,0 +1,144 @@ +import re +import json +from typing import Annotated, List, Literal, Tuple, Union +from annotated_types import Len +from loguru import logger +from pydantic import BaseModel, ValidationError +from anthropic.types import Message, TextBlock + +from dendrite.browser.async_api._core.models.api_config import APIConfig + +from .agent import ( + AnthropicAgent, +) +from .prompts import ( + SEGMENT_PROMPT, +) + + + + +class SegmentAgentSuccessResponse(BaseModel): + reason: str + status: Literal["success"] + d_id: Annotated[List[str], Len(min_length=1)] + index: int = 99999 # placeholder since the agent doesn't output this + + +class SegmentAgentFailureResponse(BaseModel): + reason: str + status: Literal["failed", "loading", "impossible"] + index: int = 99999 # placeholder since the agent doesn't output this + + +SegmentAgentReponseType = Union[ + SegmentAgentSuccessResponse, SegmentAgentFailureResponse +] + + +def parse_claude_result(result: Message, index: int) -> SegmentAgentReponseType: + json_pattern = r"```json(.*?)```" + model = None + + if len(result.content) == 0 or not isinstance(result.content[0], TextBlock): + return SegmentAgentFailureResponse( + reason="No content from agent", status="failed", index=index + ) + + text = result.content[0].text + + if text is None: + return SegmentAgentFailureResponse( + reason="No content", status="failed", index=index + ) + + json_matches = re.findall(json_pattern, text, re.DOTALL) + + if not json_matches: + return SegmentAgentFailureResponse( + reason="No JSON matches", status="failed", index=index + ) + + json_match = json_matches[0] + try: + json_data = json.loads(json_match) + if "d_id" in json_data and "reason" in json_data: + ids = json_data["d_id"] + if len(ids) == 0: + logger.warning( + f"Success message was output, but no d_ids provided: {json_data}" + ) + return SegmentAgentFailureResponse( + reason="No d_ids provided", status="failed", index=index + ) + + model = SegmentAgentSuccessResponse( + reason=json_data["reason"], + status="success", + d_id=json_data["d_id"], + ) + except json.JSONDecodeError as e: + raise ValueError(f"Failed to decode JSON: {e}") + + if model is None: + try: + model = SegmentAgentFailureResponse.model_validate_json(json_matches[0]) + except ValidationError as e: + logger.bind(json=json_matches[0]).error( + f"Failed to parse JSON: {e}", + ) + model = SegmentAgentFailureResponse( + reason="Failed to parse JSON", status="failed", index=index + ) + + model.index = index + return model + + + +async def extract_relevant_d_ids( + prompt: str, + segments: List[str], + api_config: APIConfig, + index: int, +) -> Tuple[int, int, SegmentAgentReponseType]: + agent = AnthropicAgent( + "claude-3-haiku-20240307", api_config, system_message=SEGMENT_PROMPT + ) + message = "" + for segment in segments: + message += ( + f"""###### SEGMENT ######\n\n{segment}\n\n###### SEGMENT END ######\n\n""" + ) + + message += f"Can you get the d_ids of the elements that match the following description:\n\n{prompt} element\n\nIf you've selected an element you should NOT select another element that is a child of the element you've selected. It is important that you follow this." + message += """\nOutput how you think. Think step by step. if there are multiple candidate elements return all of them. Don't make up d-id for elements if they are not present/don't match the description. Limit your reasoning to 2-3 sentences\nOnly include the json block – don't output an array, only ONE object.""" + + max_retries = 3 + for attempt in range(max_retries): + res = await agent.add_message(message) + if res is None: + message = "I didn't receive a response. Please try again." + continue + + try: + parsed_res = parse_claude_result(res, index) + # If we successfully parsed the result, return it + completion = res.usage.output_tokens if res.usage else 0 + prompt_token = res.usage.input_tokens if res.usage else 0 + return (prompt_token, completion, parsed_res) + except Exception as e: + # If we encounter a ValueError, ask the agent to correct its output + logger.warning(f"Error in segment agent: {e}") + message = f"An exception occurred in your output: {e}\n\nPlease correct your output and try again. Ensure you're providing a valid JSON response." + + # If we've exhausted all retries, return a failure response + return ( + 0, + 0, + SegmentAgentFailureResponse( + reason="Max retries reached without successful parsing", + status="failed", + index=index, + ), + ) diff --git a/dendrite/logic/local/get_element/agents/select_agent.py b/dendrite/logic/local/get_element/agents/select_agent.py new file mode 100644 index 0000000..f377960 --- /dev/null +++ b/dendrite/logic/local/get_element/agents/select_agent.py @@ -0,0 +1,108 @@ +import re +from typing import List, Optional, Tuple +from pydantic import BaseModel +from anthropic.types import Message, TextBlock +from dendrite.browser.async_api._core.models.api_config import APIConfig +from openai.types.chat import ChatCompletion + +from dendrite.browser._common.types import Status +from .agent import ( + AnthropicAgent, +) +from .prompts import ( + SELECT_PROMPT, +) +from ..dom import SelectedTag + + + +class SelectAgentResponse(BaseModel): + reason: str + d_ids: Optional[List[str]] = None + status: Status + + +async def select_best_tag( + expanded_html_tree: str, + tags: List[SelectedTag], + prompt: str, + api_config: APIConfig, + time_since_frame_navigated: Optional[float], + return_several: bool = False, +) -> Tuple[int, int, Optional[SelectAgentResponse]]: + + agent = AnthropicAgent( + "claude-3-5-sonnet-20241022", api_config, system_message=SELECT_PROMPT + ) + + message = f"\n{prompt}\n" + + tags_str = "\n".join([f"d-id: {tag.d_id} - reason: '{tag.reason}'" for tag in tags]) + + message += f"""\n\nA smaller and less intelligent AI agent has combed through the html document and found these elements that seems to match the element description:\n\n{tags_str}\n\nThis agent is very primitive however, so don't blindly trust it. Make sure you carefully look at this truncated version of the html document and do some proper reasoning in which you consider the different potential elements:\n\n```html\n{expanded_html_tree}\n```\n""" + + if return_several: + message += f"""Please look at the HTML Tree and output a list of d-ids that matches the ELEMENT_DESCRIPTION.""" + else: + message += f"""Please look at the HTML Tree and output the best d-id that matches the ELEMENT_DESCRIPTION. Only return ONE d-id.""" + + if time_since_frame_navigated: + message += f"""\n\nThis page was first loaded {round(time_since_frame_navigated, 2)} second(s) ago. If the page is blank or the data is not available on the current page it could be because the page is still loading.\n\nDon't return an element that isn't what the user asked for, in this case it is better to return `status: impossible` or `status: loading` if you think the page is still loading.""" + + res = await agent.add_message(message) + # messages = agent.dump_messages() + # with open("select_agent_messages.json", "w") as f: + # f.write(messages) + + parsed = await parse_select_response(res) + + # token_usage = res.usage.input_tokens + res.usage.output_tokens + return (0, 0, parsed) + + +async def parse_select_response(result: Message) -> Optional[SelectAgentResponse]: + json_pattern = r"```json(.*?)```" + + if not isinstance(result.content[0], TextBlock): + return None + + text = result.content[0].text + json_matches = re.findall(json_pattern, text, re.DOTALL) + + if not json_matches: + return None + + try: + model = SelectAgentResponse.model_validate_json(json_matches[0]) + except Exception as e: + model = None + + return model + + +async def parse_openai_select_response( + result: ChatCompletion, +) -> Optional[SelectAgentResponse]: + json_pattern = r"```json(.*?)```" + + # Ensure the result has a message and content field + if len(result.choices) == 0 or result.choices[0].message.content is None: + return None + + # Extract the text content + text = result.choices[0].message.content + + # Find JSON formatted code block in the response text + json_matches = re.findall(json_pattern, text, re.DOTALL) + + if not json_matches: + return None + + try: + # Attempt to validate and parse the JSON match + model = SelectAgentResponse.model_validate_json(json_matches[0]) + except Exception as e: + # In case of any error during parsing + model = None + + return model diff --git a/dendrite/logic/local/get_element/cached_selector.py b/dendrite/logic/local/get_element/cached_selector.py new file mode 100644 index 0000000..beab8ce --- /dev/null +++ b/dendrite/logic/local/get_element/cached_selector.py @@ -0,0 +1,56 @@ +from datetime import datetime +from typing import List, Optional +from urllib.parse import urlparse +from bs4 import BeautifulSoup +from loguru import logger +from pydantic import BaseModel + +from dendrite.logic.interfaces.cache import CacheProtocol + + +class Selector(BaseModel): + selector: str + prompt: str + url: str + netloc: str + created_at: str + + +def deserialize_selector(selector_from_db) -> Selector: + return Selector( + selector=str(selector_from_db["selector"]), + prompt=selector_from_db.get("prompts", ""), + url=selector_from_db.get("url", ""), + netloc=selector_from_db.get("netloc", ""), + created_at=selector_from_db.get("created_at", ""), + ) + + +async def get_selector_from_db( + url: str, prompt: str, cache: CacheProtocol[Selector] +) -> Optional[Selector]: + + netloc = urlparse(url).netloc + + return cache.get({"netloc": netloc, "prompt": prompt}) + +async def add_selector_in_db( + prompt: str, bs4_selector: str, url: str +): + + created_at = datetime.now().isoformat() + netloc = urlparse(url).netloc + selector: Selector = Selector( + prompt=prompt, + selector=bs4_selector, + url=url, + netloc=netloc, + created_at=created_at, + ) + serialized = selector.model_dump() + # res = await selector_collection.insert_one(serialized) + # return str(res.inserted_id) + + + + diff --git a/dendrite/logic/local/get_element/dom.py b/dendrite/logic/local/get_element/dom.py new file mode 100644 index 0000000..071b6e7 --- /dev/null +++ b/dendrite/logic/local/get_element/dom.py @@ -0,0 +1,338 @@ +import copy +from dataclasses import dataclass +from collections import deque +from typing import List, Optional, Union, overload +from bs4 import BeautifulSoup, Comment, Doctype, NavigableString, Tag +from ..dom.truncate import truncate_and_remove_whitespace, truncate_long_string_w_words + + + +@overload +def shorten_attr_val(value: str, limit: int = 50) -> str: ... + + +@overload +def shorten_attr_val(value: List[str], limit: int = 50) -> List[str]: ... + + +def shorten_attr_val( + value: Union[str, List[str]], limit: int = 50 +) -> Union[str, List[str]]: + if isinstance(value, str): + return value[:limit] + + char_count = sum(map(len, value)) + if char_count <= limit: + return value + + while len(value) > 1 and char_count > limit: + char_count -= len(value.pop()) + + if len(value) == 1: + return value[0][:limit] + + return value + + +def clear_attrs(element: Tag): + + salient_attributes = [ + "d-id", + "class", + "id", + "type", + "alt", + "aria-describedby", + "aria-label", + "contenteditable", + "aria-role", + "input-checked", + "label", + "name", + "option_selected", + "placeholder", + "readonly", + "text-value", + "title", + "value", + "href", + "role", + "action", + "method", + ] + attrs = { + attr: shorten_attr_val(value, limit=200) + for attr, value in element.attrs.items() + if attr in salient_attributes + } + element.attrs = attrs + + +def strip_soup(soup: BeautifulSoup) -> BeautifulSoup: + # Create a copy of the soup to avoid modifying the original + stripped_soup = BeautifulSoup(str(soup), "html.parser") + + for tag in stripped_soup( + [ + "head", + "script", + "style", + "path", + "polygon", + "defs", + "br", + "Doctype", + ] # add noscript? + ): + tag.extract() + + # Remove comments + comments = stripped_soup.find_all(text=lambda text: isinstance(text, Comment)) + for comment in comments: + comment.extract() + + # Clear non-salient attributes + for element in stripped_soup.find_all(True): + if isinstance(element, Doctype): + element.extract() + else: + clear_attrs(element) + + return stripped_soup + +import copy + + + +def remove_hidden_elements(soup: BeautifulSoup): + # data-hidden is added by DendriteBrowser when an element is not visible + new_soup = copy.copy(soup) + elems = new_soup.find_all(attrs={"data-hidden": True}) + for elem in elems: + elem.extract() + return new_soup + + +# Define a threshold (e.g., 30% of the total document size) +def calculate_size(element): + as_str = str(element) + return len(as_str) + + +def format_tag(node: Union[BeautifulSoup, Tag]): + opening_tag = f"<{node.name}" + + # Add all attributes to the opening tag + for attr, value in node.attrs.items(): + opening_tag += f' {attr}="{value}"' + + # Close the opening tag + opening_tag += ">" + return opening_tag + + +@dataclass +class SegmentGroup: + node: List[Union[BeautifulSoup, Tag, str]] + parents: List[Union[BeautifulSoup, Tag]] + idx: int + size: int + order: int = 0 + + +def hanifi_segment( + node: Union[BeautifulSoup, Tag], + threshold, + num_parents: int, +) -> List[List[str]]: + segment_groups = _new_segment_tree( + node, threshold, num_parents, 0, deque(maxlen=num_parents) + ) + return group_segments(segment_groups, threshold * 1.1) + + +def group_segments(segments: List[SegmentGroup], threshold: int) -> List[List[str]]: + grouped_segments: List[List[str]] = [] + current_group: List[str] = [] + current_size = 0 + + for segment in segments: + # If adding the current segment doesn't exceed the threshold + if current_size + segment.size <= threshold: + current_group.append(reconstruct_html(segment)) + current_size += segment.size + else: + # Add the current group to the grouped_segments + grouped_segments.append(current_group) + # Start a new group with the current segment + current_group = [reconstruct_html(segment)] + current_size = segment.size + + # Add the last group if it's not empty + if current_group: + grouped_segments.append(current_group) + + return grouped_segments + + +def reconstruct_html(segment_group: SegmentGroup) -> str: + # Initialize an empty list to build the HTML parts + html_parts = [] + + # If the index is not 0, add "..." before the first sibling node + if segment_group.idx != 0: + html_parts.append("...") + + # Add the string representation of each node in the segment group + for node in segment_group.node: + html_parts.append(str(node)) + + # Combine the node HTML parts + nodes_html = "\n".join(html_parts) + + # Build the HTML by wrapping the nodes_html within the parents + for parent in reversed(segment_group.parents): + # Get the opening tag with attributes + attrs = "".join([f' {k}="{v}"' for k, v in parent.attrs.items()]) + opening_tag = f"<{parent.name}{attrs}>" + closing_tag = f"" + # Wrap the current nodes_html within this parent + nodes_html = f"{opening_tag}\n{nodes_html}\n{closing_tag}" + + # Use BeautifulSoup to parse and prettify the final HTML + soup = BeautifulSoup(nodes_html, "html.parser") + return soup.prettify() + + +def _new_segment_tree( + node: Union[BeautifulSoup, Tag], + threshold: int, + num_parents: int, + index, + queue: deque, +) -> List[SegmentGroup]: + + result_nodes = [] + idx = 0 + current_group: Optional[SegmentGroup] = None + queue.append(node) + for child in node.children: # type: ignore + + if isinstance(child, (NavigableString, Tag)): + size = 0 + if isinstance(child, NavigableString): + child = str(child) + size = len(child) + if size > threshold: + truncated = truncate_long_string_w_words( + child, max_len_start=threshold // 4, max_len_end=threshold // 4 + ) + result_nodes.append( + SegmentGroup( + node=[truncated], + parents=list(queue.copy()), + idx=idx, + size=size, + ) + ) + idx += 1 + continue + + elif isinstance(child, Tag): + size = calculate_size(child) + if size > threshold: + result_nodes.extend( + _new_segment_tree( + child, threshold, num_parents, idx, queue.copy() + ) + ) + idx += 1 + continue + + if current_group is not None: + if current_group.size + size < threshold: + current_group.node.append(child) + current_group.size += size + else: + result_nodes.append(current_group) + # **Create a new current_group with the current child** + current_group = SegmentGroup( + node=[child], parents=list(queue.copy()), idx=idx, size=size + ) + idx += 1 + continue + + # **Initialize current_group if it's None** + current_group = SegmentGroup( + node=[child], parents=list(queue.copy()), idx=idx, size=size + ) + idx += 1 + + if current_group is not None: + result_nodes.append(current_group) + + return result_nodes + + +@dataclass +class SelectedTag: + d_id: str + reason: str + index: int # index of the segment the tag belongs in + + +def expand_tags(soup: BeautifulSoup, tags: List[SelectedTag]) -> Optional[str]: + + target_d_ids = {tag.d_id for tag in tags} + target_elements = soup.find_all( + lambda tag: tag.has_attr("d-id") and tag["d-id"] in target_d_ids + ) + + if len(target_elements) == 0: + return None + + parents_list = [] + for element in target_elements: + parents = list(element.parents) + parents_list.append(parents) + + all_parent_d_ids = frozenset( + d_id + for parents in parents_list + for parent in parents + if isinstance(parent, Tag) and parent.has_attr("d-id") + for d_id in [parent.get("d-id")] + ) + + def traverse_and_simplify(element): + if isinstance(element, Tag): + d_id = element.get("d-id", "") + if element in target_elements: + # Add comments to mark the selected element + element.insert_before(Comment(f"SELECTED ELEMENT START ({d_id})")) + element.insert_after(Comment(f"SELECTED ELEMENT END ({d_id})")) + + # If element is too large, continue traversing since we don't want to display large elements + if len(str(element)) > 40000: + for child in list(element.children): + if isinstance(child, Tag): + traverse_and_simplify(child) + return + elif d_id in all_parent_d_ids or element.name == "body": + for child in list(element.children): + if isinstance(child, Tag): + traverse_and_simplify(child) + elif isinstance(element, Tag) and element.name != "body": + try: + truncated_text = truncate_and_remove_whitespace( + element.get_text(), max_len_start=200, max_len_end=200 + ) + element.replace_with(truncated_text) + except ValueError: + element.replace_with("...") + + soup_copy = copy.copy(soup) + traverse_and_simplify(soup_copy.body) + simplified_html = soup_copy.prettify() + + return simplified_html diff --git a/dendrite/logic/local/get_element/hanifi_search.py b/dendrite/logic/local/get_element/hanifi_search.py new file mode 100644 index 0000000..b10c715 --- /dev/null +++ b/dendrite/logic/local/get_element/hanifi_search.py @@ -0,0 +1,193 @@ +import asyncio +from typing import Any, Coroutine, List, Optional, Tuple, Union +from bs4 import BeautifulSoup, Tag + +from dendrite.browser.async_api._core.models.api_config import APIConfig + + +from .agents import select_agent +from .agents import segment_agent + +from .agents.segment_agent import ( + SegmentAgentFailureResponse, + SegmentAgentReponseType, + SegmentAgentSuccessResponse, + +) +from .dom import ( + SelectedTag, + expand_tags, + hanifi_segment, + strip_soup, +) +from .models import ( + Interactable, +) + + +async def get_expanded_dom( + soup: BeautifulSoup, prompt: str, api_config: APIConfig +) -> Optional[Tuple[str, List[SegmentAgentReponseType], List[SelectedTag]]]: + + new_nodes = hanifi_segment(soup, 6000, 3) + tags = await get_relevant_tags(prompt, new_nodes, api_config, soup) + + succesful_d_ids = [ + (tag.d_id, tag.index, tag.reason) + for tag in tags + if isinstance(tag, SegmentAgentSuccessResponse) + ] + + flat_list = [ + SelectedTag( + d_id, + reason=segment_d_ids[2], + index=segment_d_ids[1], + ) + for segment_d_ids in succesful_d_ids + for d_id in segment_d_ids[0] + ] + dom = expand_tags(soup, flat_list) + if dom is None: + return None + return dom, tags, flat_list + + +async def hanifi_search( + soup: BeautifulSoup, + prompt: str, + api_config: APIConfig, + time_since_frame_navigated: Optional[float] = None, + return_several: bool = False, +) -> List[Interactable]: + + stripped_soup = strip_soup(soup) + expand_res = await get_expanded_dom(stripped_soup, prompt, api_config) + + if expand_res is None: + return [ + Interactable(status="failed", reason="No element found when expanding HTML") + ] + + expanded, tags, flat_list = expand_res + + failed_messages = [] + succesful_tags: List[SegmentAgentSuccessResponse] = [] + for tag in tags: + if isinstance(tag, SegmentAgentFailureResponse): + failed_messages.append(tag) + else: + succesful_tags.append(tag) + + if len(succesful_tags) == 0: + return [Interactable(status="failed", reason="No relevant tags found in DOM")] + + (input_token, output_token, res) = await select_agent.select_best_tag( + expanded, + flat_list, + prompt, + api_config, + time_since_frame_navigated, + return_several, + ) + + if not res: + return [Interactable(status="failed", reason="Failed to get interactable")] + + if res.d_ids: + if return_several: + return [ + Interactable(status=res.status, dendrite_id=d_id, reason=res.reason) + for d_id in res.d_ids + ] + else: + return [ + Interactable( + status=res.status, dendrite_id=res.d_ids[0], reason=res.reason + ) + ] + + return [Interactable(status=res.status, dendrite_id=None, reason=res.reason)] + + +async def get_relevant_tags( + prompt: str, + segments: List[List[str]], + api_config: APIConfig, + soup: BeautifulSoup, +) -> List[SegmentAgentReponseType]: + tasks: List[Coroutine[Any, Any, Tuple[int, int, SegmentAgentReponseType]]] = [] + for index, segment in enumerate(segments): + tasks.append( + segment_agent.extract_relevant_d_ids(prompt, segment, api_config, index) + ) + + results: List[Tuple[int, int, SegmentAgentReponseType]] = await asyncio.gather( + *tasks + ) + if results is None: + return + + tokens_prompt = 0 + tokens_completion = 0 + + tags = [] + for res in results: + tokens_prompt += res[0] + tokens_completion += res[1] + tags.append(res[2]) + + return tags + + +def get_if_one_tag( + lst: List[SegmentAgentSuccessResponse], +) -> Optional[SegmentAgentSuccessResponse]: + curr_item = None + for item in lst: + if isinstance(item, SegmentAgentSuccessResponse): + d_id_count = len(item.d_id) + if d_id_count > 1: # There are multiple d_ids + return None + + if curr_item is None: + curr_item = item # There should always be atleast one d_id + else: # We have already found a d_id + return None + + return curr_item + + +def process_segments( + nodes: List[Union[Tag, BeautifulSoup]], threshold: int = 5000 +) -> List[List[Union[Tag, BeautifulSoup]]]: + processed_segments: List[List[Union[Tag, BeautifulSoup]]] = [] + grouped_segments: List[Union[Tag, BeautifulSoup]] = [] + current_len = 0 + for index, node in enumerate(nodes): + node_len = len(str(node)) + + if current_len + node_len > threshold: + processed_segments.append(grouped_segments) + grouped_segments = [] + current_len = 0 + + grouped_segments.append(node) + current_len += node_len + + if grouped_segments: + processed_segments.append(grouped_segments) + + return processed_segments + + +def dump_processed_segments(processed_segments: List[List[Union[Tag, BeautifulSoup]]]): + for index, processed_segement in enumerate(processed_segments): + with open(f"processed_segments/segment_{index}.html", "w") as f: + f.write("######\n\n".join(map(lambda x: x.prettify(), processed_segement))) + + +def dump_nodes(nodes: List[Union[Tag, BeautifulSoup]]): + for index, node in enumerate(nodes): + with open(f"nodes/node_{index}.html", "w") as f: + f.write(node.prettify()) diff --git a/dendrite/logic/local/get_element/main.py b/dendrite/logic/local/get_element/main.py new file mode 100644 index 0000000..650b2ef --- /dev/null +++ b/dendrite/logic/local/get_element/main.py @@ -0,0 +1,101 @@ + +from typing import Optional +from anthropic import BaseModel +from bs4 import BeautifulSoup, Tag +from loguru import logger +from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.browser.async_api._core.models.page_information import PageInformation +from dendrite.browser.sync_api._api.response.get_element_response import GetElementResponse +from dendrite.logic.interfaces.cache import CacheProtocol +from dendrite.logic.local.dom.css import check_if_selector_successful, find_css_selector +from .hanifi_search import hanifi_search +from dendrite.logic.local.get_element.cached_selector import add_selector_in_db, get_selector_from_db +from dendrite.logic.local.get_element.dom import remove_hidden_elements + + +class GetElementDTO(BaseModel): + page_information: PageInformation + prompt: str + api_config: APIConfig + use_cache: bool = True + only_one: bool + force_use_cache: bool = False + + +async def process_single_prompt( + get_elements_dto: GetElementDTO, prompt: str, user_id: str, cache: Optional[CacheProtocol] = None +) -> GetElementResponse: + + soup = BeautifulSoup(get_elements_dto.page_information.raw_html, "lxml") + + if get_elements_dto.use_cache and cache: + res = await check_cache(soup, get_elements_dto.page_information.url, prompt, get_elements_dto.only_one, cache) + if res: + return res + + + if get_elements_dto.force_use_cache: + return GetElementResponse( + selectors=[], + status="failed", + message="Forced to use cache, but no cached selectors found", + used_cache=False, + ) + + soup_without_hidden_elements = remove_hidden_elements(soup) + + if get_elements_dto.only_one: + interactables_res = await hanifi_search( + soup_without_hidden_elements, + prompt, + get_elements_dto.api_config, + get_elements_dto.page_information.time_since_frame_navigated, + ) + interactable = interactables_res[0] + + if interactable.status == "success": + tag = soup.find(attrs={"d-id": interactable.dendrite_id}) + if isinstance(tag, Tag): + selector = find_css_selector(tag, soup) + await add_selector_in_db( + prompt, + bs4_selector=selector, + url=get_elements_dto.page_information.url, + ) + return GetElementResponse( + selectors=[selector], + message=interactable.reason, + status="success", + used_cache=False, + ) + else: + return GetElementResponse( + message=interactable.reason, + status=interactable.status, + used_cache=False, + ) + + +async def get_element_selector_action( + get_elements_dto: GetElementDTO, user_id: str +) -> GetElementResponse: + return await process_single_prompt( + get_elements_dto, get_elements_dto.prompt, user_id + ) + +async def check_cache(soup: BeautifulSoup, url: str, prompt: str, only_one: bool, cache: CacheProtocol) -> Optional[GetElementResponse]: + db_selectors = await get_selector_from_db( + url, prompt, cache + ) + + if db_selectors is None: + return None + + successful_selectors = [] + + if check_if_selector_successful(db_selectors.selector, soup, only_one): + return GetElementResponse( + selectors=successful_selectors, + status="success", + used_cache=True, + ) diff --git a/dendrite/logic/local/get_element/models.py b/dendrite/logic/local/get_element/models.py new file mode 100644 index 0000000..ead7476 --- /dev/null +++ b/dendrite/logic/local/get_element/models.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from typing import NamedTuple, Optional + +from dendrite.browser._common.types import Status + + +class ExpandedTag(NamedTuple): + d_id: str + html: str + + +@dataclass +class Interactable: + status: Status + reason: str + dendrite_id: Optional[str] = None diff --git a/dendrite/logic/local/llm/token_count.py b/dendrite/logic/local/llm/token_count.py new file mode 100644 index 0000000..12e6d80 --- /dev/null +++ b/dendrite/logic/local/llm/token_count.py @@ -0,0 +1,7 @@ +import tiktoken + + +def token_count(string: str, encoding_name) -> int: + encoding = tiktoken.encoding_for_model(encoding_name) + num_tokens = len(encoding.encode(string)) + return num_tokens diff --git a/dendrite/models/api_config.py b/dendrite/models/api_config.py new file mode 100644 index 0000000..c78502c --- /dev/null +++ b/dendrite/models/api_config.py @@ -0,0 +1,33 @@ +from typing import Optional + +from anthropic import BaseModel +from pydantic import model_validator + +from dendrite.browser._common._exceptions.dendrite_exception import MissingApiKeyError + + +class APIConfig(BaseModel): + """ + Configuration model for API keys used in the Dendrite SDK. + + Attributes: + dendrite_api_key (Optional[str]): The API key for Dendrite services. + openai_api_key (Optional[str]): The API key for OpenAI services. If you wish to use your own API key, you can do so by passing it to the Dendrite. + anthropic_api_key (Optional[str]): The API key for Anthropic services. If you wish to use your own API key, you can do so by passing it to the Dendrite. + + Raises: + ValueError: If a valid dendrite_api_key is not provided. + """ + + dendrite_api_key: Optional[str] = None + openai_api_key: Optional[str] = None + anthropic_api_key: Optional[str] = None + + @model_validator(mode="before") + def _check_api_keys(cls, values): + dendrite_api_key = values.get("dendrite_api_key") + if not dendrite_api_key: + raise MissingApiKeyError( + "A valid dendrite_api_key must be provided. Make sure you have set the DENDRITE_API_KEY environment variable or passed it to the Dendrite." + ) + return values \ No newline at end of file diff --git a/dendrite/models/dto/ask_page_dto.py b/dendrite/models/dto/ask_page_dto.py new file mode 100644 index 0000000..33a4921 --- /dev/null +++ b/dendrite/models/dto/ask_page_dto.py @@ -0,0 +1,12 @@ +from typing import Any, Optional +from pydantic import BaseModel + +from dendrite.models.page_information import PageInformation + + + +class AskPageDTO(BaseModel): + prompt: str + return_schema: Optional[Any] + page_information: PageInformation + diff --git a/dendrite/browser/async_api/_api/dto/extract_dto.py b/dendrite/models/dto/extract_dto.py similarity index 78% rename from dendrite/browser/async_api/_api/dto/extract_dto.py rename to dendrite/models/dto/extract_dto.py index 8cf1cc7..50b465a 100644 --- a/dendrite/browser/async_api/_api/dto/extract_dto.py +++ b/dendrite/models/dto/extract_dto.py @@ -1,14 +1,16 @@ import json -from typing import Any +from typing import Any, List, Optional from pydantic import BaseModel + from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation +from dendrite.models.page_information import PageInformation class ExtractDTO(BaseModel): page_information: PageInformation api_config: APIConfig prompt: str + return_data_json_schema: Any use_screenshot: bool = False use_cache: bool = True @@ -19,7 +21,7 @@ def combined_prompt(self) -> str: json_schema_prompt = ( "" - if self.return_data_json_schema is None + if self.return_data_json_schema == None else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" ) return f"Task: {self.prompt}{json_schema_prompt}" diff --git a/dendrite/browser/async_api/_api/dto/get_elements_dto.py b/dendrite/models/dto/get_elements_dto.py similarity index 65% rename from dendrite/browser/async_api/_api/dto/get_elements_dto.py rename to dendrite/models/dto/get_elements_dto.py index 636c896..e1d9bf8 100644 --- a/dendrite/browser/async_api/_api/dto/get_elements_dto.py +++ b/dendrite/models/dto/get_elements_dto.py @@ -1,8 +1,8 @@ from typing import Dict, Union from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation +from dendrite.models.page_information import PageInformation + class CheckSelectorCacheDTO(BaseModel): @@ -11,9 +11,8 @@ class CheckSelectorCacheDTO(BaseModel): class GetElementsDTO(BaseModel): - page_information: PageInformation prompt: Union[str, Dict[str, str]] - api_config: APIConfig + page_information: PageInformation use_cache: bool = True - only_one: bool force_use_cache: bool = False + only_one: bool diff --git a/dendrite/models/page_information.py b/dendrite/models/page_information.py new file mode 100644 index 0000000..8d0dae2 --- /dev/null +++ b/dendrite/models/page_information.py @@ -0,0 +1,8 @@ +from anthropic import BaseModel + + +class PageInformation(BaseModel): + url: str + raw_html: str + screenshot_base64: str + time_since_frame_navigated: float diff --git a/dendrite/browser/async_api/_api/response/extract_response.py b/dendrite/models/response/extract_page_response.py similarity index 51% rename from dendrite/browser/async_api/_api/response/extract_response.py rename to dendrite/models/response/extract_page_response.py index dd7f6d3..c714404 100644 --- a/dendrite/browser/async_api/_api/response/extract_response.py +++ b/dendrite/models/response/extract_page_response.py @@ -1,13 +1,12 @@ -from typing import Generic, Optional, TypeVar +from typing import Any, Generic, List, Optional, TypeVar from pydantic import BaseModel - -from dendrite.browser.async_api._common.status import Status +from dendrite.browser._common.types import Status T = TypeVar("T") -class ExtractResponse(BaseModel, Generic[T]): +class ExtractPageResponse(BaseModel, Generic[T]): return_data: T message: str created_script: Optional[str] = None diff --git a/poetry.lock b/poetry.lock index c6da94b..1fc589e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,142 @@ # This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +[[package]] +name = "aiohappyeyeballs" +version = "2.4.3" +description = "Happy Eyeballs for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, + {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, +] + +[[package]] +name = "aiohttp" +version = "3.10.10" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be7443669ae9c016b71f402e43208e13ddf00912f47f623ee5994e12fc7d4b3f"}, + {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7b06b7843929e41a94ea09eb1ce3927865387e3e23ebe108e0d0d09b08d25be9"}, + {file = "aiohttp-3.10.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:333cf6cf8e65f6a1e06e9eb3e643a0c515bb850d470902274239fea02033e9a8"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:274cfa632350225ce3fdeb318c23b4a10ec25c0e2c880eff951a3842cf358ac1"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9e5e4a85bdb56d224f412d9c98ae4cbd032cc4f3161818f692cd81766eee65a"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b606353da03edcc71130b52388d25f9a30a126e04caef1fd637e31683033abd"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab5a5a0c7a7991d90446a198689c0535be89bbd6b410a1f9a66688f0880ec026"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:578a4b875af3e0daaf1ac6fa983d93e0bbfec3ead753b6d6f33d467100cdc67b"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8105fd8a890df77b76dd3054cddf01a879fc13e8af576805d667e0fa0224c35d"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3bcd391d083f636c06a68715e69467963d1f9600f85ef556ea82e9ef25f043f7"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fbc6264158392bad9df19537e872d476f7c57adf718944cc1e4495cbabf38e2a"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e48d5021a84d341bcaf95c8460b152cfbad770d28e5fe14a768988c461b821bc"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2609e9ab08474702cc67b7702dbb8a80e392c54613ebe80db7e8dbdb79837c68"}, + {file = "aiohttp-3.10.10-cp310-cp310-win32.whl", hash = "sha256:84afcdea18eda514c25bc68b9af2a2b1adea7c08899175a51fe7c4fb6d551257"}, + {file = "aiohttp-3.10.10-cp310-cp310-win_amd64.whl", hash = "sha256:9c72109213eb9d3874f7ac8c0c5fa90e072d678e117d9061c06e30c85b4cf0e6"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c30a0eafc89d28e7f959281b58198a9fa5e99405f716c0289b7892ca345fe45f"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:258c5dd01afc10015866114e210fb7365f0d02d9d059c3c3415382ab633fcbcb"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:15ecd889a709b0080f02721255b3f80bb261c2293d3c748151274dfea93ac871"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3935f82f6f4a3820270842e90456ebad3af15810cf65932bd24da4463bc0a4c"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:413251f6fcf552a33c981c4709a6bba37b12710982fec8e558ae944bfb2abd38"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1720b4f14c78a3089562b8875b53e36b51c97c51adc53325a69b79b4b48ebcb"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:679abe5d3858b33c2cf74faec299fda60ea9de62916e8b67e625d65bf069a3b7"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79019094f87c9fb44f8d769e41dbb664d6e8fcfd62f665ccce36762deaa0e911"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2fb38c2ed905a2582948e2de560675e9dfbee94c6d5ccdb1301c6d0a5bf092"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a3f00003de6eba42d6e94fabb4125600d6e484846dbf90ea8e48a800430cc142"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1bbb122c557a16fafc10354b9d99ebf2f2808a660d78202f10ba9d50786384b9"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:30ca7c3b94708a9d7ae76ff281b2f47d8eaf2579cd05971b5dc681db8caac6e1"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df9270660711670e68803107d55c2b5949c2e0f2e4896da176e1ecfc068b974a"}, + {file = "aiohttp-3.10.10-cp311-cp311-win32.whl", hash = "sha256:aafc8ee9b742ce75044ae9a4d3e60e3d918d15a4c2e08a6c3c3e38fa59b92d94"}, + {file = "aiohttp-3.10.10-cp311-cp311-win_amd64.whl", hash = "sha256:362f641f9071e5f3ee6f8e7d37d5ed0d95aae656adf4ef578313ee585b585959"}, + {file = "aiohttp-3.10.10-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9294bbb581f92770e6ed5c19559e1e99255e4ca604a22c5c6397b2f9dd3ee42c"}, + {file = "aiohttp-3.10.10-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a8fa23fe62c436ccf23ff930149c047f060c7126eae3ccea005f0483f27b2e28"}, + {file = "aiohttp-3.10.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5c6a5b8c7926ba5d8545c7dd22961a107526562da31a7a32fa2456baf040939f"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:007ec22fbc573e5eb2fb7dec4198ef8f6bf2fe4ce20020798b2eb5d0abda6138"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9627cc1a10c8c409b5822a92d57a77f383b554463d1884008e051c32ab1b3742"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50edbcad60d8f0e3eccc68da67f37268b5144ecc34d59f27a02f9611c1d4eec7"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a45d85cf20b5e0d0aa5a8dca27cce8eddef3292bc29d72dcad1641f4ed50aa16"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b00807e2605f16e1e198f33a53ce3c4523114059b0c09c337209ae55e3823a8"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f2d4324a98062be0525d16f768a03e0bbb3b9fe301ceee99611dc9a7953124e6"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:438cd072f75bb6612f2aca29f8bd7cdf6e35e8f160bc312e49fbecab77c99e3a"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:baa42524a82f75303f714108fea528ccacf0386af429b69fff141ffef1c534f9"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a7d8d14fe962153fc681f6366bdec33d4356f98a3e3567782aac1b6e0e40109a"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c1277cd707c465cd09572a774559a3cc7c7a28802eb3a2a9472588f062097205"}, + {file = "aiohttp-3.10.10-cp312-cp312-win32.whl", hash = "sha256:59bb3c54aa420521dc4ce3cc2c3fe2ad82adf7b09403fa1f48ae45c0cbde6628"}, + {file = "aiohttp-3.10.10-cp312-cp312-win_amd64.whl", hash = "sha256:0e1b370d8007c4ae31ee6db7f9a2fe801a42b146cec80a86766e7ad5c4a259cf"}, + {file = "aiohttp-3.10.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ad7593bb24b2ab09e65e8a1d385606f0f47c65b5a2ae6c551db67d6653e78c28"}, + {file = "aiohttp-3.10.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1eb89d3d29adaf533588f209768a9c02e44e4baf832b08118749c5fad191781d"}, + {file = "aiohttp-3.10.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3fe407bf93533a6fa82dece0e74dbcaaf5d684e5a51862887f9eaebe6372cd79"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50aed5155f819873d23520919e16703fc8925e509abbb1a1491b0087d1cd969e"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f05e9727ce409358baa615dbeb9b969db94324a79b5a5cea45d39bdb01d82e6"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dffb610a30d643983aeb185ce134f97f290f8935f0abccdd32c77bed9388b42"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa6658732517ddabe22c9036479eabce6036655ba87a0224c612e1ae6af2087e"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:741a46d58677d8c733175d7e5aa618d277cd9d880301a380fd296975a9cdd7bc"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e00e3505cd80440f6c98c6d69269dcc2a119f86ad0a9fd70bccc59504bebd68a"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ffe595f10566f8276b76dc3a11ae4bb7eba1aac8ddd75811736a15b0d5311414"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bdfcf6443637c148c4e1a20c48c566aa694fa5e288d34b20fcdc58507882fed3"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d183cf9c797a5291e8301790ed6d053480ed94070637bfaad914dd38b0981f67"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:77abf6665ae54000b98b3c742bc6ea1d1fb31c394bcabf8b5d2c1ac3ebfe7f3b"}, + {file = "aiohttp-3.10.10-cp313-cp313-win32.whl", hash = "sha256:4470c73c12cd9109db8277287d11f9dd98f77fc54155fc71a7738a83ffcc8ea8"}, + {file = "aiohttp-3.10.10-cp313-cp313-win_amd64.whl", hash = "sha256:486f7aabfa292719a2753c016cc3a8f8172965cabb3ea2e7f7436c7f5a22a151"}, + {file = "aiohttp-3.10.10-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1b66ccafef7336a1e1f0e389901f60c1d920102315a56df85e49552308fc0486"}, + {file = "aiohttp-3.10.10-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:acd48d5b80ee80f9432a165c0ac8cbf9253eaddb6113269a5e18699b33958dbb"}, + {file = "aiohttp-3.10.10-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3455522392fb15ff549d92fbf4b73b559d5e43dc522588f7eb3e54c3f38beee7"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45c3b868724137f713a38376fef8120c166d1eadd50da1855c112fe97954aed8"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:da1dee8948d2137bb51fbb8a53cce6b1bcc86003c6b42565f008438b806cccd8"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5ce2ce7c997e1971b7184ee37deb6ea9922ef5163c6ee5aa3c274b05f9e12fa"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28529e08fde6f12eba8677f5a8608500ed33c086f974de68cc65ab218713a59d"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7db54c7914cc99d901d93a34704833568d86c20925b2762f9fa779f9cd2e70f"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03a42ac7895406220124c88911ebee31ba8b2d24c98507f4a8bf826b2937c7f2"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7e338c0523d024fad378b376a79faff37fafb3c001872a618cde1d322400a572"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:038f514fe39e235e9fef6717fbf944057bfa24f9b3db9ee551a7ecf584b5b480"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:64f6c17757251e2b8d885d728b6433d9d970573586a78b78ba8929b0f41d045a"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:93429602396f3383a797a2a70e5f1de5df8e35535d7806c9f91df06f297e109b"}, + {file = "aiohttp-3.10.10-cp38-cp38-win32.whl", hash = "sha256:c823bc3971c44ab93e611ab1a46b1eafeae474c0c844aff4b7474287b75fe49c"}, + {file = "aiohttp-3.10.10-cp38-cp38-win_amd64.whl", hash = "sha256:54ca74df1be3c7ca1cf7f4c971c79c2daf48d9aa65dea1a662ae18926f5bc8ce"}, + {file = "aiohttp-3.10.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:01948b1d570f83ee7bbf5a60ea2375a89dfb09fd419170e7f5af029510033d24"}, + {file = "aiohttp-3.10.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9fc1500fd2a952c5c8e3b29aaf7e3cc6e27e9cfc0a8819b3bce48cc1b849e4cc"}, + {file = "aiohttp-3.10.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f614ab0c76397661b90b6851a030004dac502e48260ea10f2441abd2207fbcc7"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00819de9e45d42584bed046314c40ea7e9aea95411b38971082cad449392b08c"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05646ebe6b94cc93407b3bf34b9eb26c20722384d068eb7339de802154d61bc5"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:998f3bd3cfc95e9424a6acd7840cbdd39e45bc09ef87533c006f94ac47296090"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9010c31cd6fa59438da4e58a7f19e4753f7f264300cd152e7f90d4602449762"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ea7ffc6d6d6f8a11e6f40091a1040995cdff02cfc9ba4c2f30a516cb2633554"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ef9c33cc5cbca35808f6c74be11eb7f5f6b14d2311be84a15b594bd3e58b5527"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ce0cdc074d540265bfeb31336e678b4e37316849d13b308607efa527e981f5c2"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:597a079284b7ee65ee102bc3a6ea226a37d2b96d0418cc9047490f231dc09fe8"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:7789050d9e5d0c309c706953e5e8876e38662d57d45f936902e176d19f1c58ab"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e7f8b04d83483577fd9200461b057c9f14ced334dcb053090cea1da9c8321a91"}, + {file = "aiohttp-3.10.10-cp39-cp39-win32.whl", hash = "sha256:c02a30b904282777d872266b87b20ed8cc0d1501855e27f831320f471d54d983"}, + {file = "aiohttp-3.10.10-cp39-cp39-win_amd64.whl", hash = "sha256:edfe3341033a6b53a5c522c802deb2079eee5cbfbb0af032a55064bd65c73a23"}, + {file = "aiohttp-3.10.10.tar.gz", hash = "sha256:0631dd7c9f0822cc61c88586ca76d5b5ada26538097d0f1df510b082bad3411a"}, +] + +[package.dependencies] +aiohappyeyeballs = ">=2.3.0" +aiosignal = ">=1.1.2" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.12.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + [[package]] name = "annotated-types" version = "0.7.0" @@ -47,6 +184,36 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "attrs" +version = "24.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, +] + +[package.extras] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] + [[package]] name = "autopep8" version = "2.3.1" @@ -154,6 +321,120 @@ files = [ {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] +[[package]] +name = "charset-normalizer" +version = "3.4.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, +] + [[package]] name = "click" version = "8.1.7" @@ -194,6 +475,17 @@ files = [ graph = ["objgraph (>=1.7.2)"] profile = ["gprof2dot (>=2022.7.29)"] +[[package]] +name = "distro" +version = "1.9.0" +description = "Distro - an OS platform information API" +optional = false +python-versions = ">=3.6" +files = [ + {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, + {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, +] + [[package]] name = "exceptiongroup" version = "1.2.2" @@ -208,6 +500,22 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "filelock" +version = "3.16.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, + {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, +] + +[package.extras] +docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] +typing = ["typing-extensions (>=4.12.2)"] + [[package]] name = "flake8" version = "7.1.1" @@ -224,6 +532,146 @@ mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.12.0,<2.13.0" pyflakes = ">=3.2.0,<3.3.0" +[[package]] +name = "frozenlist" +version = "1.5.0" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"}, + {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"}, + {file = "frozenlist-1.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15538c0cbf0e4fa11d1e3a71f823524b0c46299aed6e10ebb4c2089abd8c3bec"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e79225373c317ff1e35f210dd5f1344ff31066ba8067c307ab60254cd3a78ad5"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9272fa73ca71266702c4c3e2d4a28553ea03418e591e377a03b8e3659d94fa76"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:498524025a5b8ba81695761d78c8dd7382ac0b052f34e66939c42df860b8ff17"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92b5278ed9d50fe610185ecd23c55d8b307d75ca18e94c0e7de328089ac5dcba"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f3c8c1dacd037df16e85227bac13cca58c30da836c6f936ba1df0c05d046d8d"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2ac49a9bedb996086057b75bf93538240538c6d9b38e57c82d51f75a73409d2"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e66cc454f97053b79c2ab09c17fbe3c825ea6b4de20baf1be28919460dd7877f"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3ba5f9a0dfed20337d3e966dc359784c9f96503674c2faf015f7fe8e96798c"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6321899477db90bdeb9299ac3627a6a53c7399c8cd58d25da094007402b039ab"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76e4753701248476e6286f2ef492af900ea67d9706a0155335a40ea21bf3b2f5"}, + {file = "frozenlist-1.5.0-cp310-cp310-win32.whl", hash = "sha256:977701c081c0241d0955c9586ffdd9ce44f7a7795df39b9151cd9a6fd0ce4cfb"}, + {file = "frozenlist-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:189f03b53e64144f90990d29a27ec4f7997d91ed3d01b51fa39d2dbe77540fd4"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fd74520371c3c4175142d02a976aee0b4cb4a7cc912a60586ffd8d5929979b30"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f3f7a0fbc219fb4455264cae4d9f01ad41ae6ee8524500f381de64ffaa077d5"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f47c9c9028f55a04ac254346e92977bf0f166c483c74b4232bee19a6697e4778"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0996c66760924da6e88922756d99b47512a71cfd45215f3570bf1e0b694c206a"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2fe128eb4edeabe11896cb6af88fca5346059f6c8d807e3b910069f39157869"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a8ea951bbb6cacd492e3948b8da8c502a3f814f5d20935aae74b5df2b19cf3d"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de537c11e4aa01d37db0d403b57bd6f0546e71a82347a97c6a9f0dcc532b3a45"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c2623347b933fcb9095841f1cc5d4ff0b278addd743e0e966cb3d460278840d"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cee6798eaf8b1416ef6909b06f7dc04b60755206bddc599f52232606e18179d3"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f5f9da7f5dbc00a604fe74aa02ae7c98bcede8a3b8b9666f9f86fc13993bc71a"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:90646abbc7a5d5c7c19461d2e3eeb76eb0b204919e6ece342feb6032c9325ae9"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bdac3c7d9b705d253b2ce370fde941836a5f8b3c5c2b8fd70940a3ea3af7f4f2"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03d33c2ddbc1816237a67f66336616416e2bbb6beb306e5f890f2eb22b959cdf"}, + {file = "frozenlist-1.5.0-cp311-cp311-win32.whl", hash = "sha256:237f6b23ee0f44066219dae14c70ae38a63f0440ce6750f868ee08775073f942"}, + {file = "frozenlist-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:0cc974cc93d32c42e7b0f6cf242a6bd941c57c61b618e78b6c0a96cb72788c1d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f"}, + {file = "frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8"}, + {file = "frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7a1a048f9215c90973402e26c01d1cff8a209e1f1b53f72b95c13db61b00f953"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dd47a5181ce5fcb463b5d9e17ecfdb02b678cca31280639255ce9d0e5aa67af0"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1431d60b36d15cda188ea222033eec8e0eab488f39a272461f2e6d9e1a8e63c2"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6482a5851f5d72767fbd0e507e80737f9c8646ae7fd303def99bfe813f76cf7f"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44c49271a937625619e862baacbd037a7ef86dd1ee215afc298a417ff3270608"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:12f78f98c2f1c2429d42e6a485f433722b0061d5c0b0139efa64f396efb5886b"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce3aa154c452d2467487765e3adc730a8c153af77ad84096bc19ce19a2400840"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b7dc0c4338e6b8b091e8faf0db3168a37101943e687f373dce00959583f7439"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45e0896250900b5aa25180f9aec243e84e92ac84bd4a74d9ad4138ef3f5c97de"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:561eb1c9579d495fddb6da8959fd2a1fca2c6d060d4113f5844b433fc02f2641"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:df6e2f325bfee1f49f81aaac97d2aa757c7646534a06f8f577ce184afe2f0a9e"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:140228863501b44b809fb39ec56b5d4071f4d0aa6d216c19cbb08b8c5a7eadb9"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7707a25d6a77f5d27ea7dc7d1fc608aa0a478193823f88511ef5e6b8a48f9d03"}, + {file = "frozenlist-1.5.0-cp313-cp313-win32.whl", hash = "sha256:31a9ac2b38ab9b5a8933b693db4939764ad3f299fcaa931a3e605bc3460e693c"}, + {file = "frozenlist-1.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:11aabdd62b8b9c4b84081a3c246506d1cddd2dd93ff0ad53ede5defec7886b28"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:dd94994fc91a6177bfaafd7d9fd951bc8689b0a98168aa26b5f543868548d3ca"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0da8bbec082bf6bf18345b180958775363588678f64998c2b7609e34719b10"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:73f2e31ea8dd7df61a359b731716018c2be196e5bb3b74ddba107f694fbd7604"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:828afae9f17e6de596825cf4228ff28fbdf6065974e5ac1410cecc22f699d2b3"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1577515d35ed5649d52ab4319db757bb881ce3b2b796d7283e6634d99ace307"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2150cc6305a2c2ab33299453e2968611dacb970d2283a14955923062c8d00b10"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a72b7a6e3cd2725eff67cd64c8f13335ee18fc3c7befc05aed043d24c7b9ccb9"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c16d2fa63e0800723139137d667e1056bee1a1cf7965153d2d104b62855e9b99"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:17dcc32fc7bda7ce5875435003220a457bcfa34ab7924a49a1c19f55b6ee185c"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:97160e245ea33d8609cd2b8fd997c850b56db147a304a262abc2b3be021a9171"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f1e6540b7fa044eee0bb5111ada694cf3dc15f2b0347ca125ee9ca984d5e9e6e"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:91d6c171862df0a6c61479d9724f22efb6109111017c87567cfeb7b5d1449fdf"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c1fac3e2ace2eb1052e9f7c7db480818371134410e1f5c55d65e8f3ac6d1407e"}, + {file = "frozenlist-1.5.0-cp38-cp38-win32.whl", hash = "sha256:b97f7b575ab4a8af9b7bc1d2ef7f29d3afee2226bd03ca3875c16451ad5a7723"}, + {file = "frozenlist-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:374ca2dabdccad8e2a76d40b1d037f5bd16824933bf7bcea3e59c891fd4a0923"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9bbcdfaf4af7ce002694a4e10a0159d5a8d20056a12b05b45cea944a4953f972"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1893f948bf6681733aaccf36c5232c231e3b5166d607c5fa77773611df6dc336"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2b5e23253bb709ef57a8e95e6ae48daa9ac5f265637529e4ce6b003a37b2621f"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f253985bb515ecd89629db13cb58d702035ecd8cfbca7d7a7e29a0e6d39af5f"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04a5c6babd5e8fb7d3c871dc8b321166b80e41b637c31a995ed844a6139942b6"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9fe0f1c29ba24ba6ff6abf688cb0b7cf1efab6b6aa6adc55441773c252f7411"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:226d72559fa19babe2ccd920273e767c96a49b9d3d38badd7c91a0fdeda8ea08"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b731db116ab3aedec558573c1a5eec78822b32292fe4f2f0345b7f697745c2"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:366d8f93e3edfe5a918c874702f78faac300209a4d5bf38352b2c1bdc07a766d"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1b96af8c582b94d381a1c1f51ffaedeb77c821c690ea5f01da3d70a487dd0a9b"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:c03eff4a41bd4e38415cbed054bbaff4a075b093e2394b6915dca34a40d1e38b"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:50cf5e7ee9b98f22bdecbabf3800ae78ddcc26e4a435515fc72d97903e8488e0"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1e76bfbc72353269c44e0bc2cfe171900fbf7f722ad74c9a7b638052afe6a00c"}, + {file = "frozenlist-1.5.0-cp39-cp39-win32.whl", hash = "sha256:666534d15ba8f0fda3f53969117383d5dc021266b3c1a42c9ec4855e4b58b9d3"}, + {file = "frozenlist-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:5c28f4b5dbef8a0d8aad0d4de24d1e9e981728628afaf4ea0792f5d0939372f0"}, + {file = "frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3"}, + {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, +] + +[[package]] +name = "fsspec" +version = "2024.10.0" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, + {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dev = ["pre-commit", "ruff"] +doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] +test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] +test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] +tqdm = ["tqdm"] + [[package]] name = "greenlet" version = "3.1.1" @@ -367,6 +815,40 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "huggingface-hub" +version = "0.26.2" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "huggingface_hub-0.26.2-py3-none-any.whl", hash = "sha256:98c2a5a8e786c7b2cb6fdeb2740893cba4d53e312572ed3d8afafda65b128c46"}, + {file = "huggingface_hub-0.26.2.tar.gz", hash = "sha256:b100d853465d965733964d123939ba287da60a547087783ddff8a323f340332b"}, +] + +[package.dependencies] +filelock = "*" +fsspec = ">=2023.5.0" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +hf-transfer = ["hf-transfer (>=0.1.4)"] +inference = ["aiohttp"] +quality = ["libcst (==1.4.0)", "mypy (==1.5.1)", "ruff (>=0.5.0)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +tensorflow-testing = ["keras (<3.0)", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +torch = ["safetensors[torch]", "torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] + [[package]] name = "idna" version = "3.10" @@ -381,6 +863,29 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +[[package]] +name = "importlib-metadata" +version = "8.5.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, + {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, +] + +[package.dependencies] +zipp = ">=3.20" + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +perf = ["ipython"] +test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +type = ["pytest-mypy"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -406,6 +911,179 @@ files = [ [package.extras] colors = ["colorama (>=0.4.6)"] +[[package]] +name = "jinja2" +version = "3.1.4" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jiter" +version = "0.7.1" +description = "Fast iterable JSON parser." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jiter-0.7.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:262e96d06696b673fad6f257e6a0abb6e873dc22818ca0e0600f4a1189eb334f"}, + {file = "jiter-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be6de02939aac5be97eb437f45cfd279b1dc9de358b13ea6e040e63a3221c40d"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935f10b802bc1ce2b2f61843e498c7720aa7f4e4bb7797aa8121eab017293c3d"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9cd3cccccabf5064e4bb3099c87bf67db94f805c1e62d1aefd2b7476e90e0ee2"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4aa919ebfc5f7b027cc368fe3964c0015e1963b92e1db382419dadb098a05192"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ae2d01e82c94491ce4d6f461a837f63b6c4e6dd5bb082553a70c509034ff3d4"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f9568cd66dbbdab67ae1b4c99f3f7da1228c5682d65913e3f5f95586b3cb9a9"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ecbf4e20ec2c26512736284dc1a3f8ed79b6ca7188e3b99032757ad48db97dc"}, + {file = "jiter-0.7.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b1a0508fddc70ce00b872e463b387d49308ef02b0787992ca471c8d4ba1c0fa1"}, + {file = "jiter-0.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f84c9996664c460f24213ff1e5881530abd8fafd82058d39af3682d5fd2d6316"}, + {file = "jiter-0.7.1-cp310-none-win32.whl", hash = "sha256:c915e1a1960976ba4dfe06551ea87063b2d5b4d30759012210099e712a414d9f"}, + {file = "jiter-0.7.1-cp310-none-win_amd64.whl", hash = "sha256:75bf3b7fdc5c0faa6ffffcf8028a1f974d126bac86d96490d1b51b3210aa0f3f"}, + {file = "jiter-0.7.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ad04a23a91f3d10d69d6c87a5f4471b61c2c5cd6e112e85136594a02043f462c"}, + {file = "jiter-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e47a554de88dff701226bb5722b7f1b6bccd0b98f1748459b7e56acac2707a5"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e44fff69c814a2e96a20b4ecee3e2365e9b15cf5fe4e00869d18396daa91dab"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:df0a1d05081541b45743c965436f8b5a1048d6fd726e4a030113a2699a6046ea"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f22cf8f236a645cb6d8ffe2a64edb5d2b66fb148bf7c75eea0cb36d17014a7bc"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da8589f50b728ea4bf22e0632eefa125c8aa9c38ed202a5ee6ca371f05eeb3ff"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f20de711224f2ca2dbb166a8d512f6ff48c9c38cc06b51f796520eb4722cc2ce"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8a9803396032117b85ec8cbf008a54590644a062fedd0425cbdb95e4b2b60479"}, + {file = "jiter-0.7.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3d8bae77c82741032e9d89a4026479061aba6e646de3bf5f2fc1ae2bbd9d06e0"}, + {file = "jiter-0.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3dc9939e576bbc68c813fc82f6620353ed68c194c7bcf3d58dc822591ec12490"}, + {file = "jiter-0.7.1-cp311-none-win32.whl", hash = "sha256:f7605d24cd6fab156ec89e7924578e21604feee9c4f1e9da34d8b67f63e54892"}, + {file = "jiter-0.7.1-cp311-none-win_amd64.whl", hash = "sha256:f3ea649e7751a1a29ea5ecc03c4ada0a833846c59c6da75d747899f9b48b7282"}, + {file = "jiter-0.7.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ad36a1155cbd92e7a084a568f7dc6023497df781adf2390c345dd77a120905ca"}, + {file = "jiter-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7ba52e6aaed2dc5c81a3d9b5e4ab95b039c4592c66ac973879ba57c3506492bb"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b7de0b6f6728b678540c7927587e23f715284596724be203af952418acb8a2d"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9463b62bd53c2fb85529c700c6a3beb2ee54fde8bef714b150601616dcb184a6"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:627164ec01d28af56e1f549da84caf0fe06da3880ebc7b7ee1ca15df106ae172"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25d0e5bf64e368b0aa9e0a559c3ab2f9b67e35fe7269e8a0d81f48bbd10e8963"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c244261306f08f8008b3087059601997016549cb8bb23cf4317a4827f07b7d74"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7ded4e4b75b68b843b7cea5cd7c55f738c20e1394c68c2cb10adb655526c5f1b"}, + {file = "jiter-0.7.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:80dae4f1889b9d09e5f4de6b58c490d9c8ce7730e35e0b8643ab62b1538f095c"}, + {file = "jiter-0.7.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5970cf8ec943b51bce7f4b98d2e1ed3ada170c2a789e2db3cb484486591a176a"}, + {file = "jiter-0.7.1-cp312-none-win32.whl", hash = "sha256:701d90220d6ecb3125d46853c8ca8a5bc158de8c49af60fd706475a49fee157e"}, + {file = "jiter-0.7.1-cp312-none-win_amd64.whl", hash = "sha256:7824c3ecf9ecf3321c37f4e4d4411aad49c666ee5bc2a937071bdd80917e4533"}, + {file = "jiter-0.7.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:097676a37778ba3c80cb53f34abd6943ceb0848263c21bf423ae98b090f6c6ba"}, + {file = "jiter-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3298af506d4271257c0a8f48668b0f47048d69351675dd8500f22420d4eec378"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12fd88cfe6067e2199964839c19bd2b422ca3fd792949b8f44bb8a4e7d21946a"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dacca921efcd21939123c8ea8883a54b9fa7f6545c8019ffcf4f762985b6d0c8"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de3674a5fe1f6713a746d25ad9c32cd32fadc824e64b9d6159b3b34fd9134143"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65df9dbae6d67e0788a05b4bad5706ad40f6f911e0137eb416b9eead6ba6f044"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ba9a358d59a0a55cccaa4957e6ae10b1a25ffdabda863c0343c51817610501d"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:576eb0f0c6207e9ede2b11ec01d9c2182973986514f9c60bc3b3b5d5798c8f50"}, + {file = "jiter-0.7.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:e550e29cdf3577d2c970a18f3959e6b8646fd60ef1b0507e5947dc73703b5627"}, + {file = "jiter-0.7.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:81d968dbf3ce0db2e0e4dec6b0a0d5d94f846ee84caf779b07cab49f5325ae43"}, + {file = "jiter-0.7.1-cp313-none-win32.whl", hash = "sha256:f892e547e6e79a1506eb571a676cf2f480a4533675f834e9ae98de84f9b941ac"}, + {file = "jiter-0.7.1-cp313-none-win_amd64.whl", hash = "sha256:0302f0940b1455b2a7fb0409b8d5b31183db70d2b07fd177906d83bf941385d1"}, + {file = "jiter-0.7.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:c65a3ce72b679958b79d556473f192a4dfc5895e8cc1030c9f4e434690906076"}, + {file = "jiter-0.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e80052d3db39f9bb8eb86d207a1be3d9ecee5e05fdec31380817f9609ad38e60"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70a497859c4f3f7acd71c8bd89a6f9cf753ebacacf5e3e799138b8e1843084e3"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c1288bc22b9e36854a0536ba83666c3b1fb066b811019d7b682c9cf0269cdf9f"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b096ca72dd38ef35675e1d3b01785874315182243ef7aea9752cb62266ad516f"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8dbbd52c50b605af13dbee1a08373c520e6fcc6b5d32f17738875847fea4e2cd"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af29c5c6eb2517e71ffa15c7ae9509fa5e833ec2a99319ac88cc271eca865519"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f114a4df1e40c03c0efbf974b376ed57756a1141eb27d04baee0680c5af3d424"}, + {file = "jiter-0.7.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:191fbaee7cf46a9dd9b817547bf556facde50f83199d07fc48ebeff4082f9df4"}, + {file = "jiter-0.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e2b445e5ee627fb4ee6bbceeb486251e60a0c881a8e12398dfdff47c56f0723"}, + {file = "jiter-0.7.1-cp38-none-win32.whl", hash = "sha256:47ac4c3cf8135c83e64755b7276339b26cd3c7ddadf9e67306ace4832b283edf"}, + {file = "jiter-0.7.1-cp38-none-win_amd64.whl", hash = "sha256:60b49c245cd90cde4794f5c30f123ee06ccf42fb8730a019a2870cd005653ebd"}, + {file = "jiter-0.7.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8f212eeacc7203256f526f550d105d8efa24605828382cd7d296b703181ff11d"}, + {file = "jiter-0.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d9e247079d88c00e75e297e6cb3a18a039ebcd79fefc43be9ba4eb7fb43eb726"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0aacaa56360139c53dcf352992b0331f4057a0373bbffd43f64ba0c32d2d155"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bc1b55314ca97dbb6c48d9144323896e9c1a25d41c65bcb9550b3e0c270ca560"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f281aae41b47e90deb70e7386558e877a8e62e1693e0086f37d015fa1c102289"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:93c20d2730a84d43f7c0b6fb2579dc54335db742a59cf9776d0b80e99d587382"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e81ccccd8069110e150613496deafa10da2f6ff322a707cbec2b0d52a87b9671"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0a7d5e85766eff4c9be481d77e2226b4c259999cb6862ccac5ef6621d3c8dcce"}, + {file = "jiter-0.7.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f52ce5799df5b6975439ecb16b1e879d7655e1685b6e3758c9b1b97696313bfb"}, + {file = "jiter-0.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e0c91a0304373fdf97d56f88356a010bba442e6d995eb7773cbe32885b71cdd8"}, + {file = "jiter-0.7.1-cp39-none-win32.whl", hash = "sha256:5c08adf93e41ce2755970e8aa95262298afe2bf58897fb9653c47cd93c3c6cdc"}, + {file = "jiter-0.7.1-cp39-none-win_amd64.whl", hash = "sha256:6592f4067c74176e5f369228fb2995ed01400c9e8e1225fb73417183a5e635f0"}, + {file = "jiter-0.7.1.tar.gz", hash = "sha256:448cf4f74f7363c34cdef26214da527e8eeffd88ba06d0b80b485ad0667baf5d"}, +] + +[[package]] +name = "json-repair" +version = "0.30.1" +description = "A package to repair broken json strings" +optional = false +python-versions = ">=3.9" +files = [ + {file = "json_repair-0.30.1-py3-none-any.whl", hash = "sha256:6fa8a05d246e282df2f812fa542bd837d671d7774eaae11191aabaac97d41e33"}, + {file = "json_repair-0.30.1.tar.gz", hash = "sha256:5f075c4e3b098d78fb6cd60c34aec07a4517f14e9d423ad5364214b0e870e218"}, +] + +[[package]] +name = "jsonschema" +version = "4.23.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, + {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +jsonschema-specifications = ">=2023.03.6" +referencing = ">=0.28.4" +rpds-py = ">=0.7.1" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"] + +[[package]] +name = "jsonschema-specifications" +version = "2024.10.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.9" +files = [ + {file = "jsonschema_specifications-2024.10.1-py3-none-any.whl", hash = "sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf"}, + {file = "jsonschema_specifications-2024.10.1.tar.gz", hash = "sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272"}, +] + +[package.dependencies] +referencing = ">=0.31.0" + +[[package]] +name = "litellm" +version = "1.52.6" +description = "Library to easily interface with LLM API providers" +optional = false +python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" +files = [ + {file = "litellm-1.52.6-py3-none-any.whl", hash = "sha256:9b3e9fb51f7e2a3cc8b50997b346c55aae9435a138d9a656f18e262750a1bfe1"}, + {file = "litellm-1.52.6.tar.gz", hash = "sha256:d67c653f97bd07f503b975c167de1e25632b7bc6bb3c008c46921e4acc81ec60"}, +] + +[package.dependencies] +aiohttp = "*" +click = "*" +importlib-metadata = ">=6.8.0" +jinja2 = ">=3.1.2,<4.0.0" +jsonschema = ">=4.22.0,<5.0.0" +openai = ">=1.54.0" +pydantic = ">=2.0.0,<3.0.0" +python-dotenv = ">=0.2.0" +requests = ">=2.31.0,<3.0.0" +tiktoken = ">=0.7.0" +tokenizers = "*" + +[package.extras] +extra-proxy = ["azure-identity (>=1.15.0,<2.0.0)", "azure-keyvault-secrets (>=4.8.0,<5.0.0)", "google-cloud-kms (>=2.21.3,<3.0.0)", "prisma (==0.11.0)", "resend (>=0.8.0,<0.9.0)"] +proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "cryptography (>=42.0.5,<43.0.0)", "fastapi (>=0.111.0,<0.112.0)", "fastapi-sso (>=0.10.0,<0.11.0)", "gunicorn (>=22.0.0,<23.0.0)", "orjson (>=3.9.7,<4.0.0)", "pynacl (>=1.5.0,<2.0.0)", "python-multipart (>=0.0.9,<0.0.10)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"] + [[package]] name = "loguru" version = "0.7.2" @@ -593,6 +1271,76 @@ files = [ beautifulsoup4 = ">=4.9,<5" six = ">=1.15,<2" +[[package]] +name = "markupsafe" +version = "3.0.2" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.9" +files = [ + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, +] + [[package]] name = "mccabe" version = "0.7.0" @@ -604,6 +1352,110 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "multidict" +version = "6.1.0" +description = "multidict implementation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, + {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, + {file = "multidict-6.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a114d03b938376557927ab23f1e950827c3b893ccb94b62fd95d430fd0e5cf53"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1c416351ee6271b2f49b56ad7f308072f6f44b37118d69c2cad94f3fa8a40d5"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b5d83030255983181005e6cfbac1617ce9746b219bc2aad52201ad121226581"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e97b5e938051226dc025ec80980c285b053ffb1e25a3db2a3aa3bc046bf7f56"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d618649d4e70ac6efcbba75be98b26ef5078faad23592f9b51ca492953012429"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10524ebd769727ac77ef2278390fb0068d83f3acb7773792a5080f2b0abf7748"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ff3827aef427c89a25cc96ded1759271a93603aba9fb977a6d264648ebf989db"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06809f4f0f7ab7ea2cabf9caca7d79c22c0758b58a71f9d32943ae13c7ace056"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f179dee3b863ab1c59580ff60f9d99f632f34ccb38bf67a33ec6b3ecadd0fd76"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:aaed8b0562be4a0876ee3b6946f6869b7bcdb571a5d1496683505944e268b160"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c8b88a2ccf5493b6c8da9076fb151ba106960a2df90c2633f342f120751a9e7"}, + {file = "multidict-6.1.0-cp310-cp310-win32.whl", hash = "sha256:4a9cb68166a34117d6646c0023c7b759bf197bee5ad4272f420a0141d7eb03a0"}, + {file = "multidict-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:20b9b5fbe0b88d0bdef2012ef7dee867f874b72528cf1d08f1d59b0e3850129d"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753"}, + {file = "multidict-6.1.0-cp311-cp311-win32.whl", hash = "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80"}, + {file = "multidict-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3"}, + {file = "multidict-6.1.0-cp312-cp312-win32.whl", hash = "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133"}, + {file = "multidict-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d569388c381b24671589335a3be6e1d45546c2988c2ebe30fdcada8457a31008"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f90c822a402cb865e396a504f9fc8173ef34212a342d92e362ca498cad308e28"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b225d95519a5bf73860323e633a664b0d85ad3d5bede6d30d95b35d4dfe8805b"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:23bfd518810af7de1116313ebd9092cb9aa629beb12f6ed631ad53356ed6b86c"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c09fcfdccdd0b57867577b719c69e347a436b86cd83747f179dbf0cc0d4c1f3"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf6bea52ec97e95560af5ae576bdac3aa3aae0b6758c6efa115236d9e07dae44"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57feec87371dbb3520da6192213c7d6fc892d5589a93db548331954de8248fd2"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0c3f390dc53279cbc8ba976e5f8035eab997829066756d811616b652b00a23a3"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:59bfeae4b25ec05b34f1956eaa1cb38032282cd4dfabc5056d0a1ec4d696d3aa"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b2f59caeaf7632cc633b5cf6fc449372b83bbdf0da4ae04d5be36118e46cc0aa"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:37bb93b2178e02b7b618893990941900fd25b6b9ac0fa49931a40aecdf083fe4"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e9f48f58c2c523d5a06faea47866cd35b32655c46b443f163d08c6d0ddb17d6"}, + {file = "multidict-6.1.0-cp313-cp313-win32.whl", hash = "sha256:3a37ffb35399029b45c6cc33640a92bef403c9fd388acce75cdc88f58bd19a81"}, + {file = "multidict-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9aa71e15d9d9beaad2c6b9319edcdc0a49a43ef5c0a4c8265ca9ee7d6c67774"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:db7457bac39421addd0c8449933ac32d8042aae84a14911a757ae6ca3eef1392"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d094ddec350a2fb899fec68d8353c78233debde9b7d8b4beeafa70825f1c281a"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5845c1fd4866bb5dd3125d89b90e57ed3138241540897de748cdf19de8a2fca2"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9079dfc6a70abe341f521f78405b8949f96db48da98aeb43f9907f342f627cdc"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3914f5aaa0f36d5d60e8ece6a308ee1c9784cd75ec8151062614657a114c4478"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c08be4f460903e5a9d0f76818db3250f12e9c344e79314d1d570fc69d7f4eae4"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d093be959277cb7dee84b801eb1af388b6ad3ca6a6b6bf1ed7585895789d027d"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3702ea6872c5a2a4eeefa6ffd36b042e9773f05b1f37ae3ef7264b1163c2dcf6"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2090f6a85cafc5b2db085124d752757c9d251548cedabe9bd31afe6363e0aff2"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:f67f217af4b1ff66c68a87318012de788dd95fcfeb24cc889011f4e1c7454dfd"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:189f652a87e876098bbc67b4da1049afb5f5dfbaa310dd67c594b01c10388db6"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:6bb5992037f7a9eff7991ebe4273ea7f51f1c1c511e6a2ce511d0e7bdb754492"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f4c2b9e770c4e393876e35a7046879d195cd123b4f116d299d442b335bcd"}, + {file = "multidict-6.1.0-cp38-cp38-win32.whl", hash = "sha256:e27bbb6d14416713a8bd7aaa1313c0fc8d44ee48d74497a0ff4c3a1b6ccb5167"}, + {file = "multidict-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:22f3105d4fb15c8f57ff3959a58fcab6ce36814486500cd7485651230ad4d4ef"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4e18b656c5e844539d506a0a06432274d7bd52a7487e6828c63a63d69185626c"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a185f876e69897a6f3325c3f19f26a297fa058c5e456bfcff8015e9a27e83ae1"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab7c4ceb38d91570a650dba194e1ca87c2b543488fe9309b4212694174fd539c"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e617fb6b0b6953fffd762669610c1c4ffd05632c138d61ac7e14ad187870669c"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16e5f4bf4e603eb1fdd5d8180f1a25f30056f22e55ce51fb3d6ad4ab29f7d96f"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c035da3f544b1882bac24115f3e2e8760f10a0107614fc9839fd232200b875"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:957cf8e4b6e123a9eea554fa7ebc85674674b713551de587eb318a2df3e00255"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:483a6aea59cb89904e1ceabd2b47368b5600fb7de78a6e4a2c2987b2d256cf30"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:87701f25a2352e5bf7454caa64757642734da9f6b11384c1f9d1a8e699758057"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:682b987361e5fd7a139ed565e30d81fd81e9629acc7d925a205366877d8c8657"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce2186a7df133a9c895dea3331ddc5ddad42cdd0d1ea2f0a51e5d161e4762f28"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9f636b730f7e8cb19feb87094949ba54ee5357440b9658b2a32a5ce4bce53972"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:73eae06aa53af2ea5270cc066dcaf02cc60d2994bbb2c4ef5764949257d10f43"}, + {file = "multidict-6.1.0-cp39-cp39-win32.whl", hash = "sha256:1ca0083e80e791cffc6efce7660ad24af66c8d4079d2a750b29001b53ff59ada"}, + {file = "multidict-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:aa466da5b15ccea564bdab9c89175c762bc12825f4659c11227f515cee76fa4a"}, + {file = "multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506"}, + {file = "multidict-6.1.0.tar.gz", hash = "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -615,6 +1467,30 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "openai" +version = "1.54.4" +description = "The official Python library for the openai API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "openai-1.54.4-py3-none-any.whl", hash = "sha256:0d95cef99346bf9b6d7fbf57faf61a673924c3e34fa8af84c9ffe04660673a7e"}, + {file = "openai-1.54.4.tar.gz", hash = "sha256:50f3656e45401c54e973fa05dc29f3f0b0d19348d685b2f7ddb4d92bf7b1b6bf"}, +] + +[package.dependencies] +anyio = ">=3.5.0,<5" +distro = ">=1.7.0,<2" +httpx = ">=0.23.0,<1" +jiter = ">=0.4.0,<1" +pydantic = ">=1.9.0,<3" +sniffio = "*" +tqdm = ">4" +typing-extensions = ">=4.11,<5" + +[package.extras] +datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] + [[package]] name = "packaging" version = "24.1" @@ -648,6 +1524,98 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] +[[package]] +name = "pillow" +version = "11.0.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pillow-11.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6619654954dc4936fcff82db8eb6401d3159ec6be81e33c6000dfd76ae189947"}, + {file = "pillow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b3c5ac4bed7519088103d9450a1107f76308ecf91d6dabc8a33a2fcfb18d0fba"}, + {file = "pillow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a65149d8ada1055029fcb665452b2814fe7d7082fcb0c5bed6db851cb69b2086"}, + {file = "pillow-11.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88a58d8ac0cc0e7f3a014509f0455248a76629ca9b604eca7dc5927cc593c5e9"}, + {file = "pillow-11.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c26845094b1af3c91852745ae78e3ea47abf3dbcd1cf962f16b9a5fbe3ee8488"}, + {file = "pillow-11.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1a61b54f87ab5786b8479f81c4b11f4d61702830354520837f8cc791ebba0f5f"}, + {file = "pillow-11.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:674629ff60030d144b7bca2b8330225a9b11c482ed408813924619c6f302fdbb"}, + {file = "pillow-11.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:598b4e238f13276e0008299bd2482003f48158e2b11826862b1eb2ad7c768b97"}, + {file = "pillow-11.0.0-cp310-cp310-win32.whl", hash = "sha256:9a0f748eaa434a41fccf8e1ee7a3eed68af1b690e75328fd7a60af123c193b50"}, + {file = "pillow-11.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a5629742881bcbc1f42e840af185fd4d83a5edeb96475a575f4da50d6ede337c"}, + {file = "pillow-11.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:ee217c198f2e41f184f3869f3e485557296d505b5195c513b2bfe0062dc537f1"}, + {file = "pillow-11.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1c1d72714f429a521d8d2d018badc42414c3077eb187a59579f28e4270b4b0fc"}, + {file = "pillow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:499c3a1b0d6fc8213519e193796eb1a86a1be4b1877d678b30f83fd979811d1a"}, + {file = "pillow-11.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8b2351c85d855293a299038e1f89db92a2f35e8d2f783489c6f0b2b5f3fe8a3"}, + {file = "pillow-11.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f4dba50cfa56f910241eb7f883c20f1e7b1d8f7d91c750cd0b318bad443f4d5"}, + {file = "pillow-11.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5ddbfd761ee00c12ee1be86c9c0683ecf5bb14c9772ddbd782085779a63dd55b"}, + {file = "pillow-11.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:45c566eb10b8967d71bf1ab8e4a525e5a93519e29ea071459ce517f6b903d7fa"}, + {file = "pillow-11.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b4fd7bd29610a83a8c9b564d457cf5bd92b4e11e79a4ee4716a63c959699b306"}, + {file = "pillow-11.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cb929ca942d0ec4fac404cbf520ee6cac37bf35be479b970c4ffadf2b6a1cad9"}, + {file = "pillow-11.0.0-cp311-cp311-win32.whl", hash = "sha256:006bcdd307cc47ba43e924099a038cbf9591062e6c50e570819743f5607404f5"}, + {file = "pillow-11.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:52a2d8323a465f84faaba5236567d212c3668f2ab53e1c74c15583cf507a0291"}, + {file = "pillow-11.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:16095692a253047fe3ec028e951fa4221a1f3ed3d80c397e83541a3037ff67c9"}, + {file = "pillow-11.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d2c0a187a92a1cb5ef2c8ed5412dd8d4334272617f532d4ad4de31e0495bd923"}, + {file = "pillow-11.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:084a07ef0821cfe4858fe86652fffac8e187b6ae677e9906e192aafcc1b69903"}, + {file = "pillow-11.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8069c5179902dcdce0be9bfc8235347fdbac249d23bd90514b7a47a72d9fecf4"}, + {file = "pillow-11.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f02541ef64077f22bf4924f225c0fd1248c168f86e4b7abdedd87d6ebaceab0f"}, + {file = "pillow-11.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fcb4621042ac4b7865c179bb972ed0da0218a076dc1820ffc48b1d74c1e37fe9"}, + {file = "pillow-11.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:00177a63030d612148e659b55ba99527803288cea7c75fb05766ab7981a8c1b7"}, + {file = "pillow-11.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8853a3bf12afddfdf15f57c4b02d7ded92c7a75a5d7331d19f4f9572a89c17e6"}, + {file = "pillow-11.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3107c66e43bda25359d5ef446f59c497de2b5ed4c7fdba0894f8d6cf3822dafc"}, + {file = "pillow-11.0.0-cp312-cp312-win32.whl", hash = "sha256:86510e3f5eca0ab87429dd77fafc04693195eec7fd6a137c389c3eeb4cfb77c6"}, + {file = "pillow-11.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:8ec4a89295cd6cd4d1058a5e6aec6bf51e0eaaf9714774e1bfac7cfc9051db47"}, + {file = "pillow-11.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:27a7860107500d813fcd203b4ea19b04babe79448268403172782754870dac25"}, + {file = "pillow-11.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bcd1fb5bb7b07f64c15618c89efcc2cfa3e95f0e3bcdbaf4642509de1942a699"}, + {file = "pillow-11.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0e038b0745997c7dcaae350d35859c9715c71e92ffb7e0f4a8e8a16732150f38"}, + {file = "pillow-11.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ae08bd8ffc41aebf578c2af2f9d8749d91f448b3bfd41d7d9ff573d74f2a6b2"}, + {file = "pillow-11.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d69bfd8ec3219ae71bcde1f942b728903cad25fafe3100ba2258b973bd2bc1b2"}, + {file = "pillow-11.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:61b887f9ddba63ddf62fd02a3ba7add935d053b6dd7d58998c630e6dbade8527"}, + {file = "pillow-11.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c6a660307ca9d4867caa8d9ca2c2658ab685de83792d1876274991adec7b93fa"}, + {file = "pillow-11.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:73e3a0200cdda995c7e43dd47436c1548f87a30bb27fb871f352a22ab8dcf45f"}, + {file = "pillow-11.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fba162b8872d30fea8c52b258a542c5dfd7b235fb5cb352240c8d63b414013eb"}, + {file = "pillow-11.0.0-cp313-cp313-win32.whl", hash = "sha256:f1b82c27e89fffc6da125d5eb0ca6e68017faf5efc078128cfaa42cf5cb38798"}, + {file = "pillow-11.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:8ba470552b48e5835f1d23ecb936bb7f71d206f9dfeee64245f30c3270b994de"}, + {file = "pillow-11.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:846e193e103b41e984ac921b335df59195356ce3f71dcfd155aa79c603873b84"}, + {file = "pillow-11.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4ad70c4214f67d7466bea6a08061eba35c01b1b89eaa098040a35272a8efb22b"}, + {file = "pillow-11.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6ec0d5af64f2e3d64a165f490d96368bb5dea8b8f9ad04487f9ab60dc4bb6003"}, + {file = "pillow-11.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c809a70e43c7977c4a42aefd62f0131823ebf7dd73556fa5d5950f5b354087e2"}, + {file = "pillow-11.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:4b60c9520f7207aaf2e1d94de026682fc227806c6e1f55bba7606d1c94dd623a"}, + {file = "pillow-11.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1e2688958a840c822279fda0086fec1fdab2f95bf2b717b66871c4ad9859d7e8"}, + {file = "pillow-11.0.0-cp313-cp313t-win32.whl", hash = "sha256:607bbe123c74e272e381a8d1957083a9463401f7bd01287f50521ecb05a313f8"}, + {file = "pillow-11.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c39ed17edea3bc69c743a8dd3e9853b7509625c2462532e62baa0732163a904"}, + {file = "pillow-11.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:75acbbeb05b86bc53cbe7b7e6fe00fbcf82ad7c684b3ad82e3d711da9ba287d3"}, + {file = "pillow-11.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:2e46773dc9f35a1dd28bd6981332fd7f27bec001a918a72a79b4133cf5291dba"}, + {file = "pillow-11.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2679d2258b7f1192b378e2893a8a0a0ca472234d4c2c0e6bdd3380e8dfa21b6a"}, + {file = "pillow-11.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eda2616eb2313cbb3eebbe51f19362eb434b18e3bb599466a1ffa76a033fb916"}, + {file = "pillow-11.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ec184af98a121fb2da42642dea8a29ec80fc3efbaefb86d8fdd2606619045d"}, + {file = "pillow-11.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:8594f42df584e5b4bb9281799698403f7af489fba84c34d53d1c4bfb71b7c4e7"}, + {file = "pillow-11.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:c12b5ae868897c7338519c03049a806af85b9b8c237b7d675b8c5e089e4a618e"}, + {file = "pillow-11.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:70fbbdacd1d271b77b7721fe3cdd2d537bbbd75d29e6300c672ec6bb38d9672f"}, + {file = "pillow-11.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5178952973e588b3f1360868847334e9e3bf49d19e169bbbdfaf8398002419ae"}, + {file = "pillow-11.0.0-cp39-cp39-win32.whl", hash = "sha256:8c676b587da5673d3c75bd67dd2a8cdfeb282ca38a30f37950511766b26858c4"}, + {file = "pillow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:94f3e1780abb45062287b4614a5bc0874519c86a777d4a7ad34978e86428b8dd"}, + {file = "pillow-11.0.0-cp39-cp39-win_arm64.whl", hash = "sha256:290f2cc809f9da7d6d622550bbf4c1e57518212da51b6a30fe8e0a270a5b78bd"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1187739620f2b365de756ce086fdb3604573337cc28a0d3ac4a01ab6b2d2a6d2"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:fbbcb7b57dc9c794843e3d1258c0fbf0f48656d46ffe9e09b63bbd6e8cd5d0a2"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d203af30149ae339ad1b4f710d9844ed8796e97fda23ffbc4cc472968a47d0b"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a0d3b115009ebb8ac3d2ebec5c2982cc693da935f4ab7bb5c8ebe2f47d36f2"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:73853108f56df97baf2bb8b522f3578221e56f646ba345a372c78326710d3830"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e58876c91f97b0952eb766123bfef372792ab3f4e3e1f1a2267834c2ab131734"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:224aaa38177597bb179f3ec87eeefcce8e4f85e608025e9cfac60de237ba6316"}, + {file = "pillow-11.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5bd2d3bdb846d757055910f0a59792d33b555800813c3b39ada1829c372ccb06"}, + {file = "pillow-11.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:375b8dd15a1f5d2feafff536d47e22f69625c1aa92f12b339ec0b2ca40263273"}, + {file = "pillow-11.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:daffdf51ee5db69a82dd127eabecce20729e21f7a3680cf7cbb23f0829189790"}, + {file = "pillow-11.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7326a1787e3c7b0429659e0a944725e1b03eeaa10edd945a86dead1913383944"}, + {file = "pillow-11.0.0.tar.gz", hash = "sha256:72bacbaf24ac003fea9bff9837d1eedb6088758d41e100c1552930151f677739"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] + [[package]] name = "platformdirs" version = "4.3.6" @@ -718,6 +1686,113 @@ tomli = {version = ">=1.2.2", markers = "python_version < \"3.11\""} [package.extras] poetry-plugin = ["poetry (>=1.0,<2.0)"] +[[package]] +name = "propcache" +version = "0.2.0" +description = "Accelerated property cache" +optional = false +python-versions = ">=3.8" +files = [ + {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"}, + {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"}, + {file = "propcache-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:33ac8f098df0585c0b53009f039dfd913b38c1d2edafed0cedcc0c32a05aa110"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e48e8875e6c13909c800fa344cd54cc4b2b0db1d5f911f840458a500fde2c2"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388f3217649d6d59292b722d940d4d2e1e6a7003259eb835724092a1cca0203a"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f571aea50ba5623c308aa146eb650eebf7dbe0fd8c5d946e28343cb3b5aad577"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3dfafb44f7bb35c0c06eda6b2ab4bfd58f02729e7c4045e179f9a861b07c9850"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3ebe9a75be7ab0b7da2464a77bb27febcb4fab46a34f9288f39d74833db7f61"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d2f0d0f976985f85dfb5f3d685697ef769faa6b71993b46b295cdbbd6be8cc37"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a3dc1a4b165283bd865e8f8cb5f0c64c05001e0718ed06250d8cac9bec115b48"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9e0f07b42d2a50c7dd2d8675d50f7343d998c64008f1da5fef888396b7f84630"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e63e3e1e0271f374ed489ff5ee73d4b6e7c60710e1f76af5f0e1a6117cd26394"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:56bb5c98f058a41bb58eead194b4db8c05b088c93d94d5161728515bd52b052b"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7665f04d0c7f26ff8bb534e1c65068409bf4687aa2534faf7104d7182debb336"}, + {file = "propcache-0.2.0-cp310-cp310-win32.whl", hash = "sha256:7cf18abf9764746b9c8704774d8b06714bcb0a63641518a3a89c7f85cc02c2ad"}, + {file = "propcache-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:cfac69017ef97db2438efb854edf24f5a29fd09a536ff3a992b75990720cdc99"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:63f13bf09cc3336eb04a837490b8f332e0db41da66995c9fd1ba04552e516354"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608cce1da6f2672a56b24a015b42db4ac612ee709f3d29f27a00c943d9e851de"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:466c219deee4536fbc83c08d09115249db301550625c7fef1c5563a584c9bc87"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2db02409338bf36590aa985a461b2c96fce91f8e7e0f14c50c5fcc4f229016"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a6ed8db0a556343d566a5c124ee483ae113acc9a557a807d439bcecc44e7dfbb"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91997d9cb4a325b60d4e3f20967f8eb08dfcb32b22554d5ef78e6fd1dda743a2"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c7dde9e533c0a49d802b4f3f218fa9ad0a1ce21f2c2eb80d5216565202acab4"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffcad6c564fe6b9b8916c1aefbb37a362deebf9394bd2974e9d84232e3e08504"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:97a58a28bcf63284e8b4d7b460cbee1edaab24634e82059c7b8c09e65284f178"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:945db8ee295d3af9dbdbb698cce9bbc5c59b5c3fe328bbc4387f59a8a35f998d"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:39e104da444a34830751715f45ef9fc537475ba21b7f1f5b0f4d71a3b60d7fe2"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c5ecca8f9bab618340c8e848d340baf68bcd8ad90a8ecd7a4524a81c1764b3db"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c436130cc779806bdf5d5fae0d848713105472b8566b75ff70048c47d3961c5b"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:191db28dc6dcd29d1a3e063c3be0b40688ed76434622c53a284e5427565bbd9b"}, + {file = "propcache-0.2.0-cp311-cp311-win32.whl", hash = "sha256:5f2564ec89058ee7c7989a7b719115bdfe2a2fb8e7a4543b8d1c0cc4cf6478c1"}, + {file = "propcache-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e2e54267980349b723cff366d1e29b138b9a60fa376664a157a342689553f71"}, + {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2"}, + {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7"}, + {file = "propcache-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2e900bad2a8456d00a113cad8c13343f3b1f327534e3589acc2219729237a2e8"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f52a68c21363c45297aca15561812d542f8fc683c85201df0bebe209e349f793"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e41d67757ff4fbc8ef2af99b338bfb955010444b92929e9e55a6d4dcc3c4f09"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a64e32f8bd94c105cc27f42d3b658902b5bcc947ece3c8fe7bc1b05982f60e89"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55346705687dbd7ef0d77883ab4f6fabc48232f587925bdaf95219bae072491e"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6994984550eaf25dd7fc7bd1b700ff45c894149341725bb4edc67f0ffa94efa4"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:56295eb1e5f3aecd516d91b00cfd8bf3a13991de5a479df9e27dd569ea23959c"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:439e76255daa0f8151d3cb325f6dd4a3e93043e6403e6491813bcaaaa8733887"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f6475a1b2ecb310c98c28d271a30df74f9dd436ee46d09236a6b750a7599ce57"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3444cdba6628accf384e349014084b1cacd866fbb88433cd9d279d90a54e0b23"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a9d9b4d0a9b38d1c391bb4ad24aa65f306c6f01b512e10a8a34a2dc5675d348"}, + {file = "propcache-0.2.0-cp312-cp312-win32.whl", hash = "sha256:69d3a98eebae99a420d4b28756c8ce6ea5a29291baf2dc9ff9414b42676f61d5"}, + {file = "propcache-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ad9c9b99b05f163109466638bd30ada1722abb01bbb85c739c50b6dc11f92dc3"}, + {file = "propcache-0.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ecddc221a077a8132cf7c747d5352a15ed763b674c0448d811f408bf803d9ad7"}, + {file = "propcache-0.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0e53cb83fdd61cbd67202735e6a6687a7b491c8742dfc39c9e01e80354956763"}, + {file = "propcache-0.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92fe151145a990c22cbccf9ae15cae8ae9eddabfc949a219c9f667877e40853d"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a21ef516d36909931a2967621eecb256018aeb11fc48656e3257e73e2e247a"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f88a4095e913f98988f5b338c1d4d5d07dbb0b6bad19892fd447484e483ba6b"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a5b3bb545ead161be780ee85a2b54fdf7092815995661947812dde94a40f6fb"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67aeb72e0f482709991aa91345a831d0b707d16b0257e8ef88a2ad246a7280bf"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c997f8c44ec9b9b0bcbf2d422cc00a1d9b9c681f56efa6ca149a941e5560da2"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2a66df3d4992bc1d725b9aa803e8c5a66c010c65c741ad901e260ece77f58d2f"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3ebbcf2a07621f29638799828b8d8668c421bfb94c6cb04269130d8de4fb7136"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1235c01ddaa80da8235741e80815ce381c5267f96cc49b1477fdcf8c047ef325"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3947483a381259c06921612550867b37d22e1df6d6d7e8361264b6d037595f44"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d5bed7f9805cc29c780f3aee05de3262ee7ce1f47083cfe9f77471e9d6777e83"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e4a91d44379f45f5e540971d41e4626dacd7f01004826a18cb048e7da7e96544"}, + {file = "propcache-0.2.0-cp313-cp313-win32.whl", hash = "sha256:f902804113e032e2cdf8c71015651c97af6418363bea8d78dc0911d56c335032"}, + {file = "propcache-0.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8f188cfcc64fb1266f4684206c9de0e80f54622c3f22a910cbd200478aeae61e"}, + {file = "propcache-0.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:53d1bd3f979ed529f0805dd35ddaca330f80a9a6d90bc0121d2ff398f8ed8861"}, + {file = "propcache-0.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:83928404adf8fb3d26793665633ea79b7361efa0287dfbd372a7e74311d51ee6"}, + {file = "propcache-0.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77a86c261679ea5f3896ec060be9dc8e365788248cc1e049632a1be682442063"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:218db2a3c297a3768c11a34812e63b3ac1c3234c3a086def9c0fee50d35add1f"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7735e82e3498c27bcb2d17cb65d62c14f1100b71723b68362872bca7d0913d90"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20a617c776f520c3875cf4511e0d1db847a076d720714ae35ffe0df3e440be68"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67b69535c870670c9f9b14a75d28baa32221d06f6b6fa6f77a0a13c5a7b0a5b9"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4569158070180c3855e9c0791c56be3ceeb192defa2cdf6a3f39e54319e56b89"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:db47514ffdbd91ccdc7e6f8407aac4ee94cc871b15b577c1c324236b013ddd04"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:2a60ad3e2553a74168d275a0ef35e8c0a965448ffbc3b300ab3a5bb9956c2162"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:662dd62358bdeaca0aee5761de8727cfd6861432e3bb828dc2a693aa0471a563"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:25a1f88b471b3bc911d18b935ecb7115dff3a192b6fef46f0bfaf71ff4f12418"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:f60f0ac7005b9f5a6091009b09a419ace1610e163fa5deaba5ce3484341840e7"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:74acd6e291f885678631b7ebc85d2d4aec458dd849b8c841b57ef04047833bed"}, + {file = "propcache-0.2.0-cp38-cp38-win32.whl", hash = "sha256:d9b6ddac6408194e934002a69bcaadbc88c10b5f38fb9307779d1c629181815d"}, + {file = "propcache-0.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:676135dcf3262c9c5081cc8f19ad55c8a64e3f7282a21266d05544450bffc3a5"}, + {file = "propcache-0.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:25c8d773a62ce0451b020c7b29a35cfbc05de8b291163a7a0f3b7904f27253e6"}, + {file = "propcache-0.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:375a12d7556d462dc64d70475a9ee5982465fbb3d2b364f16b86ba9135793638"}, + {file = "propcache-0.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1ec43d76b9677637a89d6ab86e1fef70d739217fefa208c65352ecf0282be957"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f45eec587dafd4b2d41ac189c2156461ebd0c1082d2fe7013571598abb8505d1"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc092ba439d91df90aea38168e11f75c655880c12782facf5cf9c00f3d42b562"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa1076244f54bb76e65e22cb6910365779d5c3d71d1f18b275f1dfc7b0d71b4d"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:682a7c79a2fbf40f5dbb1eb6bfe2cd865376deeac65acf9beb607505dced9e12"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e40876731f99b6f3c897b66b803c9e1c07a989b366c6b5b475fafd1f7ba3fb8"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:363ea8cd3c5cb6679f1c2f5f1f9669587361c062e4899fce56758efa928728f8"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:140fbf08ab3588b3468932974a9331aff43c0ab8a2ec2c608b6d7d1756dbb6cb"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e70fac33e8b4ac63dfc4c956fd7d85a0b1139adcfc0d964ce288b7c527537fea"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b33d7a286c0dc1a15f5fc864cc48ae92a846df287ceac2dd499926c3801054a6"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f6d5749fdd33d90e34c2efb174c7e236829147a2713334d708746e94c4bde40d"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22aa8f2272d81d9317ff5756bb108021a056805ce63dd3630e27d042c8092798"}, + {file = "propcache-0.2.0-cp39-cp39-win32.whl", hash = "sha256:73e4b40ea0eda421b115248d7e79b59214411109a5bc47d0d48e4c73e3b8fcf9"}, + {file = "propcache-0.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:9517d5e9e0731957468c29dbfd0f976736a0e55afaea843726e887f36fe017df"}, + {file = "propcache-0.2.0-py3-none-any.whl", hash = "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036"}, + {file = "propcache-0.2.0.tar.gz", hash = "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70"}, +] + [[package]] name = "pycodestyle" version = "2.12.1" @@ -744,8 +1819,8 @@ files = [ annotated-types = ">=0.6.0" pydantic-core = "2.23.4" typing-extensions = [ - {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, {version = ">=4.6.1", markers = "python_version < \"3.13\""}, + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, ] [package.extras] @@ -896,8 +1971,8 @@ files = [ astroid = ">=3.3.4,<=3.4.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ - {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, {version = ">=0.2", markers = "python_version < \"3.11\""}, + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, ] isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" @@ -980,6 +2055,20 @@ pytest = ">=8.2,<9" docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "pyyaml" version = "6.0.2" @@ -1042,6 +2131,244 @@ files = [ {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +[[package]] +name = "referencing" +version = "0.35.1" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"}, + {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + +[[package]] +name = "regex" +version = "2024.11.6" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.8" +files = [ + {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91"}, + {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0"}, + {file = "regex-2024.11.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164d8b7b3b4bcb2068b97428060b2a53be050085ef94eca7f240e7947f1b080e"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3660c82f209655a06b587d55e723f0b813d3a7db2e32e5e7dc64ac2a9e86fde"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d22326fcdef5e08c154280b71163ced384b428343ae16a5ab2b3354aed12436e"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1ac758ef6aebfc8943560194e9fd0fa18bcb34d89fd8bd2af18183afd8da3a2"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:997d6a487ff00807ba810e0f8332c18b4eb8d29463cfb7c820dc4b6e7562d0cf"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02a02d2bb04fec86ad61f3ea7f49c015a0681bf76abb9857f945d26159d2968c"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f02f93b92358ee3f78660e43b4b0091229260c5d5c408d17d60bf26b6c900e86"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:06eb1be98df10e81ebaded73fcd51989dcf534e3c753466e4b60c4697a003b67"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:040df6fe1a5504eb0f04f048e6d09cd7c7110fef851d7c567a6b6e09942feb7d"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabbfc59f2c6edba2a6622c647b716e34e8e3867e0ab975412c5c2f79b82da2"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8447d2d39b5abe381419319f942de20b7ecd60ce86f16a23b0698f22e1b70008"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:da8f5fc57d1933de22a9e23eec290a0d8a5927a5370d24bda9a6abe50683fe62"}, + {file = "regex-2024.11.6-cp310-cp310-win32.whl", hash = "sha256:b489578720afb782f6ccf2840920f3a32e31ba28a4b162e13900c3e6bd3f930e"}, + {file = "regex-2024.11.6-cp310-cp310-win_amd64.whl", hash = "sha256:5071b2093e793357c9d8b2929dfc13ac5f0a6c650559503bb81189d0a3814519"}, + {file = "regex-2024.11.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5478c6962ad548b54a591778e93cd7c456a7a29f8eca9c49e4f9a806dcc5d638"}, + {file = "regex-2024.11.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c89a8cc122b25ce6945f0423dc1352cb9593c68abd19223eebbd4e56612c5b7"}, + {file = "regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94d87b689cdd831934fa3ce16cc15cd65748e6d689f5d2b8f4f4df2065c9fa20"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1062b39a0a2b75a9c694f7a08e7183a80c63c0d62b301418ffd9c35f55aaa114"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:167ed4852351d8a750da48712c3930b031f6efdaa0f22fa1933716bfcd6bf4a3"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d548dafee61f06ebdb584080621f3e0c23fff312f0de1afc776e2a2ba99a74f"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a19f302cd1ce5dd01a9099aaa19cae6173306d1302a43b627f62e21cf18ac0"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bec9931dfb61ddd8ef2ebc05646293812cb6b16b60cf7c9511a832b6f1854b55"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9714398225f299aa85267fd222f7142fcb5c769e73d7733344efc46f2ef5cf89"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:202eb32e89f60fc147a41e55cb086db2a3f8cb82f9a9a88440dcfc5d37faae8d"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4181b814e56078e9b00427ca358ec44333765f5ca1b45597ec7446d3a1ef6e34"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:068376da5a7e4da51968ce4c122a7cd31afaaec4fccc7856c92f63876e57b51d"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f2c4184420d881a3475fb2c6f4d95d53a8d50209a2500723d831036f7c45"}, + {file = "regex-2024.11.6-cp311-cp311-win32.whl", hash = "sha256:c36f9b6f5f8649bb251a5f3f66564438977b7ef8386a52460ae77e6070d309d9"}, + {file = "regex-2024.11.6-cp311-cp311-win_amd64.whl", hash = "sha256:02e28184be537f0e75c1f9b2f8847dc51e08e6e171c6bde130b2687e0c33cf60"}, + {file = "regex-2024.11.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:52fb28f528778f184f870b7cf8f225f5eef0a8f6e3778529bdd40c7b3920796a"}, + {file = "regex-2024.11.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9"}, + {file = "regex-2024.11.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805e6b60c54bf766b251e94526ebad60b7de0c70f70a4e6210ee2891acb70bf2"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b85c2530be953a890eaffde05485238f07029600e8f098cdf1848d414a8b45e4"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb26437975da7dc36b7efad18aa9dd4ea569d2357ae6b783bf1118dabd9ea577"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abfa5080c374a76a251ba60683242bc17eeb2c9818d0d30117b4486be10c59d3"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b7fa6606c2881c1db9479b0eaa11ed5dfa11c8d60a474ff0e095099f39d98e"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c32f75920cf99fe6b6c539c399a4a128452eaf1af27f39bce8909c9a3fd8cbe"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:982e6d21414e78e1f51cf595d7f321dcd14de1f2881c5dc6a6e23bbbbd68435e"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a7c2155f790e2fb448faed6dd241386719802296ec588a8b9051c1f5c481bc29"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149f5008d286636e48cd0b1dd65018548944e495b0265b45e1bffecce1ef7f39"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e5364a4502efca094731680e80009632ad6624084aff9a23ce8c8c6820de3e51"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0a86e7eeca091c09e021db8eb72d54751e527fa47b8d5787caf96d9831bd02ad"}, + {file = "regex-2024.11.6-cp312-cp312-win32.whl", hash = "sha256:32f9a4c643baad4efa81d549c2aadefaeba12249b2adc5af541759237eee1c54"}, + {file = "regex-2024.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:a93c194e2df18f7d264092dc8539b8ffb86b45b899ab976aa15d48214138e81b"}, + {file = "regex-2024.11.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a6ba92c0bcdf96cbf43a12c717eae4bc98325ca3730f6b130ffa2e3c3c723d84"}, + {file = "regex-2024.11.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:525eab0b789891ac3be914d36893bdf972d483fe66551f79d3e27146191a37d4"}, + {file = "regex-2024.11.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:086a27a0b4ca227941700e0b31425e7a28ef1ae8e5e05a33826e17e47fbfdba0"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bde01f35767c4a7899b7eb6e823b125a64de314a8ee9791367c9a34d56af18d0"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b583904576650166b3d920d2bcce13971f6f9e9a396c673187f49811b2769dc7"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c4de13f06a0d54fa0d5ab1b7138bfa0d883220965a29616e3ea61b35d5f5fc7"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3cde6e9f2580eb1665965ce9bf17ff4952f34f5b126beb509fee8f4e994f143c"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d7f453dca13f40a02b79636a339c5b62b670141e63efd511d3f8f73fba162b3"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59dfe1ed21aea057a65c6b586afd2a945de04fc7db3de0a6e3ed5397ad491b07"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b97c1e0bd37c5cd7902e65f410779d39eeda155800b65fc4d04cc432efa9bc6e"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f9d1e379028e0fc2ae3654bac3cbbef81bf3fd571272a42d56c24007979bafb6"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:13291b39131e2d002a7940fb176e120bec5145f3aeb7621be6534e46251912c4"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f51f88c126370dcec4908576c5a627220da6c09d0bff31cfa89f2523843316d"}, + {file = "regex-2024.11.6-cp313-cp313-win32.whl", hash = "sha256:63b13cfd72e9601125027202cad74995ab26921d8cd935c25f09c630436348ff"}, + {file = "regex-2024.11.6-cp313-cp313-win_amd64.whl", hash = "sha256:2b3361af3198667e99927da8b84c1b010752fa4b1115ee30beaa332cabc3ef1a"}, + {file = "regex-2024.11.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3a51ccc315653ba012774efca4f23d1d2a8a8f278a6072e29c7147eee7da446b"}, + {file = "regex-2024.11.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ad182d02e40de7459b73155deb8996bbd8e96852267879396fb274e8700190e3"}, + {file = "regex-2024.11.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ba9b72e5643641b7d41fa1f6d5abda2c9a263ae835b917348fc3c928182ad467"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40291b1b89ca6ad8d3f2b82782cc33807f1406cf68c8d440861da6304d8ffbbd"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cdf58d0e516ee426a48f7b2c03a332a4114420716d55769ff7108c37a09951bf"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a36fdf2af13c2b14738f6e973aba563623cb77d753bbbd8d414d18bfaa3105dd"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1cee317bfc014c2419a76bcc87f071405e3966da434e03e13beb45f8aced1a6"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50153825ee016b91549962f970d6a4442fa106832e14c918acd1c8e479916c4f"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea1bfda2f7162605f6e8178223576856b3d791109f15ea99a9f95c16a7636fb5"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:df951c5f4a1b1910f1a99ff42c473ff60f8225baa1cdd3539fe2819d9543e9df"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:072623554418a9911446278f16ecb398fb3b540147a7828c06e2011fa531e773"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f654882311409afb1d780b940234208a252322c24a93b442ca714d119e68086c"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89d75e7293d2b3e674db7d4d9b1bee7f8f3d1609428e293771d1a962617150cc"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:f65557897fc977a44ab205ea871b690adaef6b9da6afda4790a2484b04293a5f"}, + {file = "regex-2024.11.6-cp38-cp38-win32.whl", hash = "sha256:6f44ec28b1f858c98d3036ad5d7d0bfc568bdd7a74f9c24e25f41ef1ebfd81a4"}, + {file = "regex-2024.11.6-cp38-cp38-win_amd64.whl", hash = "sha256:bb8f74f2f10dbf13a0be8de623ba4f9491faf58c24064f32b65679b021ed0001"}, + {file = "regex-2024.11.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5704e174f8ccab2026bd2f1ab6c510345ae8eac818b613d7d73e785f1310f839"}, + {file = "regex-2024.11.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:220902c3c5cc6af55d4fe19ead504de80eb91f786dc102fbd74894b1551f095e"}, + {file = "regex-2024.11.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7e351589da0850c125f1600a4c4ba3c722efefe16b297de54300f08d734fbf"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5056b185ca113c88e18223183aa1a50e66507769c9640a6ff75859619d73957b"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e34b51b650b23ed3354b5a07aab37034d9f923db2a40519139af34f485f77d0"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5670bce7b200273eee1840ef307bfa07cda90b38ae56e9a6ebcc9f50da9c469b"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08986dce1339bc932923e7d1232ce9881499a0e02925f7402fb7c982515419ef"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93c0b12d3d3bc25af4ebbf38f9ee780a487e8bf6954c115b9f015822d3bb8e48"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:764e71f22ab3b305e7f4c21f1a97e1526a25ebdd22513e251cf376760213da13"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f056bf21105c2515c32372bbc057f43eb02aae2fda61052e2f7622c801f0b4e2"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:69ab78f848845569401469da20df3e081e6b5a11cb086de3eed1d48f5ed57c95"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:86fddba590aad9208e2fa8b43b4c098bb0ec74f15718bb6a704e3c63e2cef3e9"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:684d7a212682996d21ca12ef3c17353c021fe9de6049e19ac8481ec35574a70f"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a03e02f48cd1abbd9f3b7e3586d97c8f7a9721c436f51a5245b3b9483044480b"}, + {file = "regex-2024.11.6-cp39-cp39-win32.whl", hash = "sha256:41758407fc32d5c3c5de163888068cfee69cb4c2be844e7ac517a52770f9af57"}, + {file = "regex-2024.11.6-cp39-cp39-win_amd64.whl", hash = "sha256:b2837718570f95dd41675328e111345f9b7095d821bac435aac173ac80b19983"}, + {file = "regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "rpds-py" +version = "0.21.0" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.9" +files = [ + {file = "rpds_py-0.21.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a017f813f24b9df929674d0332a374d40d7f0162b326562daae8066b502d0590"}, + {file = "rpds_py-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:20cc1ed0bcc86d8e1a7e968cce15be45178fd16e2ff656a243145e0b439bd250"}, + {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad116dda078d0bc4886cb7840e19811562acdc7a8e296ea6ec37e70326c1b41c"}, + {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:808f1ac7cf3b44f81c9475475ceb221f982ef548e44e024ad5f9e7060649540e"}, + {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de552f4a1916e520f2703ec474d2b4d3f86d41f353e7680b597512ffe7eac5d0"}, + {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:efec946f331349dfc4ae9d0e034c263ddde19414fe5128580f512619abed05f1"}, + {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b80b4690bbff51a034bfde9c9f6bf9357f0a8c61f548942b80f7b66356508bf5"}, + {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:085ed25baac88953d4283e5b5bd094b155075bb40d07c29c4f073e10623f9f2e"}, + {file = "rpds_py-0.21.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:daa8efac2a1273eed2354397a51216ae1e198ecbce9036fba4e7610b308b6153"}, + {file = "rpds_py-0.21.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:95a5bad1ac8a5c77b4e658671642e4af3707f095d2b78a1fdd08af0dfb647624"}, + {file = "rpds_py-0.21.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3e53861b29a13d5b70116ea4230b5f0f3547b2c222c5daa090eb7c9c82d7f664"}, + {file = "rpds_py-0.21.0-cp310-none-win32.whl", hash = "sha256:ea3a6ac4d74820c98fcc9da4a57847ad2cc36475a8bd9683f32ab6d47a2bd682"}, + {file = "rpds_py-0.21.0-cp310-none-win_amd64.whl", hash = "sha256:b8f107395f2f1d151181880b69a2869c69e87ec079c49c0016ab96860b6acbe5"}, + {file = "rpds_py-0.21.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5555db3e618a77034954b9dc547eae94166391a98eb867905ec8fcbce1308d95"}, + {file = "rpds_py-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:97ef67d9bbc3e15584c2f3c74bcf064af36336c10d2e21a2131e123ce0f924c9"}, + {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab2c2a26d2f69cdf833174f4d9d86118edc781ad9a8fa13970b527bf8236027"}, + {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4e8921a259f54bfbc755c5bbd60c82bb2339ae0324163f32868f63f0ebb873d9"}, + {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a7ff941004d74d55a47f916afc38494bd1cfd4b53c482b77c03147c91ac0ac3"}, + {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5145282a7cd2ac16ea0dc46b82167754d5e103a05614b724457cffe614f25bd8"}, + {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de609a6f1b682f70bb7163da745ee815d8f230d97276db049ab447767466a09d"}, + {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40c91c6e34cf016fa8e6b59d75e3dbe354830777fcfd74c58b279dceb7975b75"}, + {file = "rpds_py-0.21.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d2132377f9deef0c4db89e65e8bb28644ff75a18df5293e132a8d67748397b9f"}, + {file = "rpds_py-0.21.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0a9e0759e7be10109645a9fddaaad0619d58c9bf30a3f248a2ea57a7c417173a"}, + {file = "rpds_py-0.21.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9e20da3957bdf7824afdd4b6eeb29510e83e026473e04952dca565170cd1ecc8"}, + {file = "rpds_py-0.21.0-cp311-none-win32.whl", hash = "sha256:f71009b0d5e94c0e86533c0b27ed7cacc1239cb51c178fd239c3cfefefb0400a"}, + {file = "rpds_py-0.21.0-cp311-none-win_amd64.whl", hash = "sha256:e168afe6bf6ab7ab46c8c375606298784ecbe3ba31c0980b7dcbb9631dcba97e"}, + {file = "rpds_py-0.21.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:30b912c965b2aa76ba5168fd610087bad7fcde47f0a8367ee8f1876086ee6d1d"}, + {file = "rpds_py-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ca9989d5d9b1b300bc18e1801c67b9f6d2c66b8fd9621b36072ed1df2c977f72"}, + {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f54e7106f0001244a5f4cf810ba8d3f9c542e2730821b16e969d6887b664266"}, + {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fed5dfefdf384d6fe975cc026886aece4f292feaf69d0eeb716cfd3c5a4dd8be"}, + {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:590ef88db231c9c1eece44dcfefd7515d8bf0d986d64d0caf06a81998a9e8cab"}, + {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f983e4c2f603c95dde63df633eec42955508eefd8d0f0e6d236d31a044c882d7"}, + {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b229ce052ddf1a01c67d68166c19cb004fb3612424921b81c46e7ea7ccf7c3bf"}, + {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ebf64e281a06c904a7636781d2e973d1f0926a5b8b480ac658dc0f556e7779f4"}, + {file = "rpds_py-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:998a8080c4495e4f72132f3d66ff91f5997d799e86cec6ee05342f8f3cda7dca"}, + {file = "rpds_py-0.21.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:98486337f7b4f3c324ab402e83453e25bb844f44418c066623db88e4c56b7c7b"}, + {file = "rpds_py-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a78d8b634c9df7f8d175451cfeac3810a702ccb85f98ec95797fa98b942cea11"}, + {file = "rpds_py-0.21.0-cp312-none-win32.whl", hash = "sha256:a58ce66847711c4aa2ecfcfaff04cb0327f907fead8945ffc47d9407f41ff952"}, + {file = "rpds_py-0.21.0-cp312-none-win_amd64.whl", hash = "sha256:e860f065cc4ea6f256d6f411aba4b1251255366e48e972f8a347cf88077b24fd"}, + {file = "rpds_py-0.21.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:ee4eafd77cc98d355a0d02f263efc0d3ae3ce4a7c24740010a8b4012bbb24937"}, + {file = "rpds_py-0.21.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:688c93b77e468d72579351a84b95f976bd7b3e84aa6686be6497045ba84be560"}, + {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c38dbf31c57032667dd5a2f0568ccde66e868e8f78d5a0d27dcc56d70f3fcd3b"}, + {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2d6129137f43f7fa02d41542ffff4871d4aefa724a5fe38e2c31a4e0fd343fb0"}, + {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:520ed8b99b0bf86a176271f6fe23024323862ac674b1ce5b02a72bfeff3fff44"}, + {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaeb25ccfb9b9014a10eaf70904ebf3f79faaa8e60e99e19eef9f478651b9b74"}, + {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af04ac89c738e0f0f1b913918024c3eab6e3ace989518ea838807177d38a2e94"}, + {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9b76e2afd585803c53c5b29e992ecd183f68285b62fe2668383a18e74abe7a3"}, + {file = "rpds_py-0.21.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5afb5efde74c54724e1a01118c6e5c15e54e642c42a1ba588ab1f03544ac8c7a"}, + {file = "rpds_py-0.21.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:52c041802a6efa625ea18027a0723676a778869481d16803481ef6cc02ea8cb3"}, + {file = "rpds_py-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee1e4fc267b437bb89990b2f2abf6c25765b89b72dd4a11e21934df449e0c976"}, + {file = "rpds_py-0.21.0-cp313-none-win32.whl", hash = "sha256:0c025820b78817db6a76413fff6866790786c38f95ea3f3d3c93dbb73b632202"}, + {file = "rpds_py-0.21.0-cp313-none-win_amd64.whl", hash = "sha256:320c808df533695326610a1b6a0a6e98f033e49de55d7dc36a13c8a30cfa756e"}, + {file = "rpds_py-0.21.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:2c51d99c30091f72a3c5d126fad26236c3f75716b8b5e5cf8effb18889ced928"}, + {file = "rpds_py-0.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cbd7504a10b0955ea287114f003b7ad62330c9e65ba012c6223dba646f6ffd05"}, + {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6dcc4949be728ede49e6244eabd04064336012b37f5c2200e8ec8eb2988b209c"}, + {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f414da5c51bf350e4b7960644617c130140423882305f7574b6cf65a3081cecb"}, + {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9afe42102b40007f588666bc7de82451e10c6788f6f70984629db193849dced1"}, + {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b929c2bb6e29ab31f12a1117c39f7e6d6450419ab7464a4ea9b0b417174f044"}, + {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8404b3717da03cbf773a1d275d01fec84ea007754ed380f63dfc24fb76ce4592"}, + {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e12bb09678f38b7597b8346983d2323a6482dcd59e423d9448108c1be37cac9d"}, + {file = "rpds_py-0.21.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:58a0e345be4b18e6b8501d3b0aa540dad90caeed814c515e5206bb2ec26736fd"}, + {file = "rpds_py-0.21.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c3761f62fcfccf0864cc4665b6e7c3f0c626f0380b41b8bd1ce322103fa3ef87"}, + {file = "rpds_py-0.21.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c2b2f71c6ad6c2e4fc9ed9401080badd1469fa9889657ec3abea42a3d6b2e1ed"}, + {file = "rpds_py-0.21.0-cp39-none-win32.whl", hash = "sha256:b21747f79f360e790525e6f6438c7569ddbfb1b3197b9e65043f25c3c9b489d8"}, + {file = "rpds_py-0.21.0-cp39-none-win_amd64.whl", hash = "sha256:0626238a43152918f9e72ede9a3b6ccc9e299adc8ade0d67c5e142d564c9a83d"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6b4ef7725386dc0762857097f6b7266a6cdd62bfd209664da6712cb26acef035"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:6bc0e697d4d79ab1aacbf20ee5f0df80359ecf55db33ff41481cf3e24f206919"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da52d62a96e61c1c444f3998c434e8b263c384f6d68aca8274d2e08d1906325c"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:98e4fe5db40db87ce1c65031463a760ec7906ab230ad2249b4572c2fc3ef1f9f"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30bdc973f10d28e0337f71d202ff29345320f8bc49a31c90e6c257e1ccef4333"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:faa5e8496c530f9c71f2b4e1c49758b06e5f4055e17144906245c99fa6d45356"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32eb88c30b6a4f0605508023b7141d043a79b14acb3b969aa0b4f99b25bc7d4a"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a89a8ce9e4e75aeb7fa5d8ad0f3fecdee813802592f4f46a15754dcb2fd6b061"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:241e6c125568493f553c3d0fdbb38c74babf54b45cef86439d4cd97ff8feb34d"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:3b766a9f57663396e4f34f5140b3595b233a7b146e94777b97a8413a1da1be18"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:af4a644bf890f56e41e74be7d34e9511e4954894d544ec6b8efe1e21a1a8da6c"}, + {file = "rpds_py-0.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3e30a69a706e8ea20444b98a49f386c17b26f860aa9245329bab0851ed100677"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:031819f906bb146561af051c7cef4ba2003d28cff07efacef59da973ff7969ba"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b876f2bc27ab5954e2fd88890c071bd0ed18b9c50f6ec3de3c50a5ece612f7a6"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc5695c321e518d9f03b7ea6abb5ea3af4567766f9852ad1560f501b17588c7b"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4de1da871b5c0fd5537b26a6fc6814c3cc05cabe0c941db6e9044ffbb12f04a"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:878f6fea96621fda5303a2867887686d7a198d9e0f8a40be100a63f5d60c88c9"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8eeec67590e94189f434c6d11c426892e396ae59e4801d17a93ac96b8c02a6c"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ff2eba7f6c0cb523d7e9cff0903f2fe1feff8f0b2ceb6bd71c0e20a4dcee271"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a429b99337062877d7875e4ff1a51fe788424d522bd64a8c0a20ef3021fdb6ed"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:d167e4dbbdac48bd58893c7e446684ad5d425b407f9336e04ab52e8b9194e2ed"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:4eb2de8a147ffe0626bfdc275fc6563aa7bf4b6db59cf0d44f0ccd6ca625a24e"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:e78868e98f34f34a88e23ee9ccaeeec460e4eaf6db16d51d7a9b883e5e785a5e"}, + {file = "rpds_py-0.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4991ca61656e3160cdaca4851151fd3f4a92e9eba5c7a530ab030d6aee96ec89"}, + {file = "rpds_py-0.21.0.tar.gz", hash = "sha256:ed6378c9d66d0de903763e7706383d60c33829581f0adff47b6535f1802fa6db"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1075,6 +2402,182 @@ files = [ {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, ] +[[package]] +name = "tiktoken" +version = "0.8.0" +description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" +optional = false +python-versions = ">=3.9" +files = [ + {file = "tiktoken-0.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b07e33283463089c81ef1467180e3e00ab00d46c2c4bbcef0acab5f771d6695e"}, + {file = "tiktoken-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9269348cb650726f44dd3bbb3f9110ac19a8dcc8f54949ad3ef652ca22a38e21"}, + {file = "tiktoken-0.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e13f37bc4ef2d012731e93e0fef21dc3b7aea5bb9009618de9a4026844e560"}, + {file = "tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f13d13c981511331eac0d01a59b5df7c0d4060a8be1e378672822213da51e0a2"}, + {file = "tiktoken-0.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6b2ddbc79a22621ce8b1166afa9f9a888a664a579350dc7c09346a3b5de837d9"}, + {file = "tiktoken-0.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d8c2d0e5ba6453a290b86cd65fc51fedf247e1ba170191715b049dac1f628005"}, + {file = "tiktoken-0.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d622d8011e6d6f239297efa42a2657043aaed06c4f68833550cac9e9bc723ef1"}, + {file = "tiktoken-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2efaf6199717b4485031b4d6edb94075e4d79177a172f38dd934d911b588d54a"}, + {file = "tiktoken-0.8.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5637e425ce1fc49cf716d88df3092048359a4b3bbb7da762840426e937ada06d"}, + {file = "tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fb0e352d1dbe15aba082883058b3cce9e48d33101bdaac1eccf66424feb5b47"}, + {file = "tiktoken-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56edfefe896c8f10aba372ab5706b9e3558e78db39dd497c940b47bf228bc419"}, + {file = "tiktoken-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:326624128590def898775b722ccc327e90b073714227175ea8febbc920ac0a99"}, + {file = "tiktoken-0.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:881839cfeae051b3628d9823b2e56b5cc93a9e2efb435f4cf15f17dc45f21586"}, + {file = "tiktoken-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe9399bdc3f29d428f16a2f86c3c8ec20be3eac5f53693ce4980371c3245729b"}, + {file = "tiktoken-0.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a58deb7075d5b69237a3ff4bb51a726670419db6ea62bdcd8bd80c78497d7ab"}, + {file = "tiktoken-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2908c0d043a7d03ebd80347266b0e58440bdef5564f84f4d29fb235b5df3b04"}, + {file = "tiktoken-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:294440d21a2a51e12d4238e68a5972095534fe9878be57d905c476017bff99fc"}, + {file = "tiktoken-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:d8f3192733ac4d77977432947d563d7e1b310b96497acd3c196c9bddb36ed9db"}, + {file = "tiktoken-0.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:02be1666096aff7da6cbd7cdaa8e7917bfed3467cd64b38b1f112e96d3b06a24"}, + {file = "tiktoken-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94ff53c5c74b535b2cbf431d907fc13c678bbd009ee633a2aca269a04389f9a"}, + {file = "tiktoken-0.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b231f5e8982c245ee3065cd84a4712d64692348bc609d84467c57b4b72dcbc5"}, + {file = "tiktoken-0.8.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4177faa809bd55f699e88c96d9bb4635d22e3f59d635ba6fd9ffedf7150b9953"}, + {file = "tiktoken-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5376b6f8dc4753cd81ead935c5f518fa0fbe7e133d9e25f648d8c4dabdd4bad7"}, + {file = "tiktoken-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:18228d624807d66c87acd8f25fc135665617cab220671eb65b50f5d70fa51f69"}, + {file = "tiktoken-0.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e17807445f0cf1f25771c9d86496bd8b5c376f7419912519699f3cc4dc5c12e"}, + {file = "tiktoken-0.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:886f80bd339578bbdba6ed6d0567a0d5c6cfe198d9e587ba6c447654c65b8edc"}, + {file = "tiktoken-0.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6adc8323016d7758d6de7313527f755b0fc6c72985b7d9291be5d96d73ecd1e1"}, + {file = "tiktoken-0.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b591fb2b30d6a72121a80be24ec7a0e9eb51c5500ddc7e4c2496516dd5e3816b"}, + {file = "tiktoken-0.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:845287b9798e476b4d762c3ebda5102be87ca26e5d2c9854002825d60cdb815d"}, + {file = "tiktoken-0.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:1473cfe584252dc3fa62adceb5b1c763c1874e04511b197da4e6de51d6ce5a02"}, + {file = "tiktoken-0.8.0.tar.gz", hash = "sha256:9ccbb2740f24542534369c5635cfd9b2b3c2490754a78ac8831d99f89f94eeb2"}, +] + +[package.dependencies] +regex = ">=2022.1.18" +requests = ">=2.26.0" + +[package.extras] +blobfile = ["blobfile (>=2)"] + +[[package]] +name = "tokenizers" +version = "0.20.3" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tokenizers-0.20.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:31ccab28dbb1a9fe539787210b0026e22debeab1662970f61c2d921f7557f7e4"}, + {file = "tokenizers-0.20.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6361191f762bda98c773da418cf511cbaa0cb8d0a1196f16f8c0119bde68ff8"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f128d5da1202b78fa0a10d8d938610472487da01b57098d48f7e944384362514"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79c4121a2e9433ad7ef0769b9ca1f7dd7fa4c0cd501763d0a030afcbc6384481"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7850fde24197fe5cd6556e2fdba53a6d3bae67c531ea33a3d7c420b90904141"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b357970c095dc134978a68c67d845a1e3803ab7c4fbb39195bde914e7e13cf8b"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a333d878c4970b72d6c07848b90c05f6b045cf9273fc2bc04a27211721ad6118"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd9fee817f655a8f50049f685e224828abfadd436b8ff67979fc1d054b435f1"}, + {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9e7816808b402129393a435ea2a509679b41246175d6e5e9f25b8692bfaa272b"}, + {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba96367db9d8a730d3a1d5996b4b7babb846c3994b8ef14008cd8660f55db59d"}, + {file = "tokenizers-0.20.3-cp310-none-win32.whl", hash = "sha256:ee31ba9d7df6a98619426283e80c6359f167e2e9882d9ce1b0254937dbd32f3f"}, + {file = "tokenizers-0.20.3-cp310-none-win_amd64.whl", hash = "sha256:a845c08fdad554fe0871d1255df85772f91236e5fd6b9287ef8b64f5807dbd0c"}, + {file = "tokenizers-0.20.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:585b51e06ca1f4839ce7759941e66766d7b060dccfdc57c4ca1e5b9a33013a90"}, + {file = "tokenizers-0.20.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61cbf11954f3b481d08723ebd048ba4b11e582986f9be74d2c3bdd9293a4538d"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef820880d5e4e8484e2fa54ff8d297bb32519eaa7815694dc835ace9130a3eea"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:67ef4dcb8841a4988cd00dd288fb95dfc8e22ed021f01f37348fd51c2b055ba9"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff1ef8bd47a02b0dc191688ccb4da53600df5d4c9a05a4b68e1e3de4823e78eb"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:444d188186eab3148baf0615b522461b41b1f0cd58cd57b862ec94b6ac9780f1"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37c04c032c1442740b2c2d925f1857885c07619224a533123ac7ea71ca5713da"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:453c7769d22231960ee0e883d1005c93c68015025a5e4ae56275406d94a3c907"}, + {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4bb31f7b2847e439766aaa9cc7bccf7ac7088052deccdb2275c952d96f691c6a"}, + {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:843729bf0f991b29655a069a2ff58a4c24375a553c70955e15e37a90dd4e045c"}, + {file = "tokenizers-0.20.3-cp311-none-win32.whl", hash = "sha256:efcce3a927b1e20ca694ba13f7a68c59b0bd859ef71e441db68ee42cf20c2442"}, + {file = "tokenizers-0.20.3-cp311-none-win_amd64.whl", hash = "sha256:88301aa0801f225725b6df5dea3d77c80365ff2362ca7e252583f2b4809c4cc0"}, + {file = "tokenizers-0.20.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:49d12a32e190fad0e79e5bdb788d05da2f20d8e006b13a70859ac47fecf6ab2f"}, + {file = "tokenizers-0.20.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:282848cacfb9c06d5e51489f38ec5aa0b3cd1e247a023061945f71f41d949d73"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abe4e08c7d0cd6154c795deb5bf81d2122f36daf075e0c12a8b050d824ef0a64"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca94fc1b73b3883c98f0c88c77700b13d55b49f1071dfd57df2b06f3ff7afd64"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef279c7e239f95c8bdd6ff319d9870f30f0d24915b04895f55b1adcf96d6c60d"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16384073973f6ccbde9852157a4fdfe632bb65208139c9d0c0bd0176a71fd67f"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:312d522caeb8a1a42ebdec87118d99b22667782b67898a76c963c058a7e41d4f"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2b7cb962564785a83dafbba0144ecb7f579f1d57d8c406cdaa7f32fe32f18ad"}, + {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:124c5882ebb88dadae1fc788a582299fcd3a8bd84fc3e260b9918cf28b8751f5"}, + {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2b6e54e71f84c4202111a489879005cb14b92616a87417f6c102c833af961ea2"}, + {file = "tokenizers-0.20.3-cp312-none-win32.whl", hash = "sha256:83d9bfbe9af86f2d9df4833c22e94d94750f1d0cd9bfb22a7bb90a86f61cdb1c"}, + {file = "tokenizers-0.20.3-cp312-none-win_amd64.whl", hash = "sha256:44def74cee574d609a36e17c8914311d1b5dbcfe37c55fd29369d42591b91cf2"}, + {file = "tokenizers-0.20.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e0b630e0b536ef0e3c8b42c685c1bc93bd19e98c0f1543db52911f8ede42cf84"}, + {file = "tokenizers-0.20.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a02d160d2b19bcbfdf28bd9a4bf11be4cb97d0499c000d95d4c4b1a4312740b6"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e3d80d89b068bc30034034b5319218c7c0a91b00af19679833f55f3becb6945"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:174a54910bed1b089226512b4458ea60d6d6fd93060254734d3bc3540953c51c"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:098b8a632b8656aa5802c46689462c5c48f02510f24029d71c208ec2c822e771"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:78c8c143e3ae41e718588281eb3e212c2b31623c9d6d40410ec464d7d6221fb5"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b26b0aadb18cd8701077362ba359a06683662d5cafe3e8e8aba10eb05c037f1"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07d7851a72717321022f3774e84aa9d595a041d643fafa2e87fbc9b18711dac0"}, + {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:bd44e48a430ada902c6266a8245f5036c4fe744fcb51f699999fbe82aa438797"}, + {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a4c186bb006ccbe1f5cc4e0380d1ce7806f5955c244074fd96abc55e27b77f01"}, + {file = "tokenizers-0.20.3-cp313-none-win32.whl", hash = "sha256:6e19e0f1d854d6ab7ea0c743d06e764d1d9a546932be0a67f33087645f00fe13"}, + {file = "tokenizers-0.20.3-cp313-none-win_amd64.whl", hash = "sha256:d50ede425c7e60966a9680d41b58b3a0950afa1bb570488e2972fa61662c4273"}, + {file = "tokenizers-0.20.3-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:9adda1ff5fb9dcdf899ceca672a4e2ce9e797adb512a6467305ca3d8bfcfbdd0"}, + {file = "tokenizers-0.20.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:6dde2cae6004ba7a3badff4a11911cae03ebf23e97eebfc0e71fef2530e5074f"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4a7fd678b35614fca708579eb95b7587a5e8a6d328171bd2488fd9f27d82be4"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b80e3c7283a01a356bd2210f53d1a4a5d32b269c2024389ed0173137708d50e"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8cc0e8176b762973758a77f0d9c4467d310e33165fb74173418ca3734944da4"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5634b2e2f5f3d2b4439d2d74066e22eb4b1f04f3fea05cb2a3c12d89b5a3bcd"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b4ba635165bc1ea46f2da8e5d80b5f70f6ec42161e38d96dbef33bb39df73964"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18e4c7c64172e7789bd8b07aa3087ea87c4c4de7e90937a2aa036b5d92332536"}, + {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1f74909ef7675c26d4095a817ec3393d67f3158ca4836c233212e5613ef640c4"}, + {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0e9b81321a1e05b16487d312b4264984513f8b4a7556229cafac6e88c2036b09"}, + {file = "tokenizers-0.20.3-cp37-none-win32.whl", hash = "sha256:ab48184cd58b4a03022a2ec75b54c9f600ffea9a733612c02325ed636f353729"}, + {file = "tokenizers-0.20.3-cp37-none-win_amd64.whl", hash = "sha256:60ac483cebee1c12c71878523e768df02fa17e4c54412966cb3ac862c91b36c1"}, + {file = "tokenizers-0.20.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3229ef103c89583d10b9378afa5d601b91e6337530a0988e17ca8d635329a996"}, + {file = "tokenizers-0.20.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6ac52cc24bad3de865c7e65b1c4e7b70d00938a8ae09a92a453b8f676e714ad5"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04627b7b502fa6a2a005e1bd446fa4247d89abcb1afaa1b81eb90e21aba9a60f"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c27ceb887f0e81a3c377eb4605dca7a95a81262761c0fba308d627b2abb98f2b"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65ab780194da4e1fcf5670523a2f377c4838ebf5249efe41fa1eddd2a84fb49d"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98d343134f47159e81f7f242264b0eb222e6b802f37173c8d7d7b64d5c9d1388"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2475bb004ab2009d29aff13b5047bfdb3d4b474f0aa9d4faa13a7f34dbbbb43"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b6583a65c01db1197c1eb36857ceba8ec329d53afadd268b42a6b04f4965724"}, + {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:62d00ba208358c037eeab7bfc00a905adc67b2d31b68ab40ed09d75881e114ea"}, + {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0fc7a39e5bedc817bda395a798dfe2d9c5f7c71153c90d381b5135a0328d9520"}, + {file = "tokenizers-0.20.3-cp38-none-win32.whl", hash = "sha256:84d40ee0f8550d64d3ea92dd7d24a8557a9172165bdb986c9fb2503b4fe4e3b6"}, + {file = "tokenizers-0.20.3-cp38-none-win_amd64.whl", hash = "sha256:205a45246ed7f1718cf3785cff88450ba603352412aaf220ace026384aa3f1c0"}, + {file = "tokenizers-0.20.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:93e37f0269a11dc3b1a953f1fca9707f0929ebf8b4063c591c71a0664219988e"}, + {file = "tokenizers-0.20.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f4cb0c614b0135e781de96c2af87e73da0389ac1458e2a97562ed26e29490d8d"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7eb2fb1c432f5746b22f8a7f09fc18c4156cb0031c77f53cb19379d82d43297a"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfa8d029bb156181b006643309d6b673615a24e4ed24cf03aa191d599b996f51"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f90549622de3bf476ad9f1dd6f3f952ec3ed6ab8615ae88ef060d0c5bfad55d"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1d469c74eebf5c43fd61cd9b030e271d17198edd7bd45392e03a3c091d7d6d4"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bee8f53b2594749f4460d53253bae55d718f04e9b633efa0f5df8938bd98e4f0"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:938441babf3e5720e4459e306ef2809fb267680df9d1ff2873458b22aef60248"}, + {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7310ab23d7b0caebecc0e8be11a1146f320f5f07284000f6ea54793e83de1b75"}, + {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:16121eb030a2b13094cfec936b0c12e8b4063c5f839591ea7d0212336d8f9921"}, + {file = "tokenizers-0.20.3-cp39-none-win32.whl", hash = "sha256:401cc21ef642ee235985d747f65e18f639464d377c70836c9003df208d582064"}, + {file = "tokenizers-0.20.3-cp39-none-win_amd64.whl", hash = "sha256:7498f3ea7746133335a6adb67a77cf77227a8b82c8483f644a2e5f86fea42b8d"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e919f2e3e68bb51dc31de4fcbbeff3bdf9c1cad489044c75e2b982a91059bd3c"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b8e9608f2773996cc272156e305bd79066163a66b0390fe21750aff62df1ac07"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39270a7050deaf50f7caff4c532c01b3c48f6608d42b3eacdebdc6795478c8df"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e005466632b1c5d2d2120f6de8aa768cc9d36cd1ab7d51d0c27a114c91a1e6ee"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a07962340b36189b6c8feda552ea1bfeee6cf067ff922a1d7760662c2ee229e5"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:55046ad3dd5f2b3c67501fcc8c9cbe3e901d8355f08a3b745e9b57894855f85b"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:efcf0eb939988b627558aaf2b9dc3e56d759cad2e0cfa04fcab378e4b48fc4fd"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f3558a7ae6a6d38a77dfce12172a1e2e1bf3e8871e744a1861cd7591ea9ebe24"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d53029fe44bc70c3ff14ef512460a0cf583495a0f8e2f4b70e26eb9438e38a9"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a2a56397b2bec5a629b516b23f0f8a3e4f978c7488d4a299980f8375954b85"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e5bfaae740ef9ece000f8a07e78ac0e2b085c5ce9648f8593ddf0243c9f76d"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fbaf3ea28fedfb2283da60e710aff25492e795a7397cad8a50f1e079b65a5a70"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c47c037116310dc976eb96b008e41b9cfaba002ed8005848d4d632ee0b7ba9ae"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c31751f0721f58f5e19bb27c1acc259aeff860d8629c4e1a900b26a1979ada8e"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:c697cbd3be7a79ea250ea5f380d6f12e534c543cfb137d5c734966b3ee4f34cc"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48971b88ef9130bf35b41b35fd857c3c4dae4a9cd7990ebc7fc03e59cc92438"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e615de179bbe060ab33773f0d98a8a8572b5883dd7dac66c1de8c056c7e748c"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da1ec842035ed9999c62e45fbe0ff14b7e8a7e02bb97688cc6313cf65e5cd755"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6ee4954c1dd23aadc27958dad759006e71659d497dcb0ef0c7c87ea992c16ebd"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3eda46ca402751ec82553a321bf35a617b76bbed7586e768c02ccacbdda94d6d"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:de082392a85eb0055cc055c535bff2f0cc15d7a000bdc36fbf601a0f3cf8507a"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c3db46cc0647bfd88263afdb739b92017a02a87ee30945cb3e86c7e25c7c9917"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a292392f24ab9abac5cfa8197e5a6208f2e43723420217e1ceba0b4ec77816ac"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dcd91f4e60f62b20d83a87a84fe062035a1e3ff49a8c2bbdeb2d441c8e311f4"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900991a2b8ee35961b1095db7e265342e0e42a84c1a594823d5ee9f8fb791958"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5a8d8261ca2133d4f98aa9627c748189502b3787537ba3d7e2beb4f7cfc5d627"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c4fd4d71e6deb6ddf99d8d0eab87d1d16f635898906e631914a9bae8ae9f2cfb"}, + {file = "tokenizers-0.20.3.tar.gz", hash = "sha256:2278b34c5d0dd78e087e1ca7f9b1dcbf129d80211afa645f214bd6e051037539"}, +] + +[package.dependencies] +huggingface-hub = ">=0.16.4,<1.0" + +[package.extras] +dev = ["tokenizers[testing]"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"] + [[package]] name = "tomli" version = "2.0.2" @@ -1097,6 +2600,27 @@ files = [ {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] +[[package]] +name = "tqdm" +version = "4.67.0" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.67.0-py3-none-any.whl", hash = "sha256:0cd8af9d56911acab92182e88d763100d4788bdf421d251616040cc4d44863be"}, + {file = "tqdm-4.67.0.tar.gz", hash = "sha256:fe5a6f95e6fe0b9755e9469b77b9c3cf850048224ecaa8293d7d2d31f97d869a"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +discord = ["requests"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -1108,6 +2632,23 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "urllib3" +version = "2.2.3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "win32-setctime" version = "1.1.0" @@ -1122,7 +2663,122 @@ files = [ [package.extras] dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] +[[package]] +name = "yarl" +version = "1.17.1" +description = "Yet another URL library" +optional = false +python-versions = ">=3.9" +files = [ + {file = "yarl-1.17.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1794853124e2f663f0ea54efb0340b457f08d40a1cef78edfa086576179c91"}, + {file = "yarl-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fbea1751729afe607d84acfd01efd95e3b31db148a181a441984ce9b3d3469da"}, + {file = "yarl-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ee427208c675f1b6e344a1f89376a9613fc30b52646a04ac0c1f6587c7e46ec"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b74ff4767d3ef47ffe0cd1d89379dc4d828d4873e5528976ced3b44fe5b0a21"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62a91aefff3d11bf60e5956d340eb507a983a7ec802b19072bb989ce120cd948"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:846dd2e1243407133d3195d2d7e4ceefcaa5f5bf7278f0a9bda00967e6326b04"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e844be8d536afa129366d9af76ed7cb8dfefec99f5f1c9e4f8ae542279a6dc3"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc7c92c1baa629cb03ecb0c3d12564f172218fb1739f54bf5f3881844daadc6d"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae3476e934b9d714aa8000d2e4c01eb2590eee10b9d8cd03e7983ad65dfbfcba"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c7e177c619342e407415d4f35dec63d2d134d951e24b5166afcdfd1362828e17"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64cc6e97f14cf8a275d79c5002281f3040c12e2e4220623b5759ea7f9868d6a5"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:84c063af19ef5130084db70ada40ce63a84f6c1ef4d3dbc34e5e8c4febb20822"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:482c122b72e3c5ec98f11457aeb436ae4aecca75de19b3d1de7cf88bc40db82f"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:380e6c38ef692b8fd5a0f6d1fa8774d81ebc08cfbd624b1bca62a4d4af2f9931"}, + {file = "yarl-1.17.1-cp310-cp310-win32.whl", hash = "sha256:16bca6678a83657dd48df84b51bd56a6c6bd401853aef6d09dc2506a78484c7b"}, + {file = "yarl-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:561c87fea99545ef7d692403c110b2f99dced6dff93056d6e04384ad3bc46243"}, + {file = "yarl-1.17.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cbad927ea8ed814622305d842c93412cb47bd39a496ed0f96bfd42b922b4a217"}, + {file = "yarl-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fca4b4307ebe9c3ec77a084da3a9d1999d164693d16492ca2b64594340999988"}, + {file = "yarl-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff5c6771c7e3511a06555afa317879b7db8d640137ba55d6ab0d0c50425cab75"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b29beab10211a746f9846baa39275e80034e065460d99eb51e45c9a9495bcca"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a52a1ffdd824fb1835272e125385c32fd8b17fbdefeedcb4d543cc23b332d74"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58c8e9620eb82a189c6c40cb6b59b4e35b2ee68b1f2afa6597732a2b467d7e8f"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d216e5d9b8749563c7f2c6f7a0831057ec844c68b4c11cb10fc62d4fd373c26d"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881764d610e3269964fc4bb3c19bb6fce55422828e152b885609ec176b41cf11"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8c79e9d7e3d8a32d4824250a9c6401194fb4c2ad9a0cec8f6a96e09a582c2cc0"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:299f11b44d8d3a588234adbe01112126010bd96d9139c3ba7b3badd9829261c3"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:cc7d768260f4ba4ea01741c1b5fe3d3a6c70eb91c87f4c8761bbcce5181beafe"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:de599af166970d6a61accde358ec9ded821234cbbc8c6413acfec06056b8e860"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2b24ec55fad43e476905eceaf14f41f6478780b870eda5d08b4d6de9a60b65b4"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9fb815155aac6bfa8d86184079652c9715c812d506b22cfa369196ef4e99d1b4"}, + {file = "yarl-1.17.1-cp311-cp311-win32.whl", hash = "sha256:7615058aabad54416ddac99ade09a5510cf77039a3b903e94e8922f25ed203d7"}, + {file = "yarl-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:14bc88baa44e1f84164a392827b5defb4fa8e56b93fecac3d15315e7c8e5d8b3"}, + {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:327828786da2006085a4d1feb2594de6f6d26f8af48b81eb1ae950c788d97f61"}, + {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc353841428d56b683a123a813e6a686e07026d6b1c5757970a877195f880c2d"}, + {file = "yarl-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c73df5b6e8fabe2ddb74876fb82d9dd44cbace0ca12e8861ce9155ad3c886139"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdff5e0995522706c53078f531fb586f56de9c4c81c243865dd5c66c132c3b5"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06157fb3c58f2736a5e47c8fcbe1afc8b5de6fb28b14d25574af9e62150fcaac"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1654ec814b18be1af2c857aa9000de7a601400bd4c9ca24629b18486c2e35463"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6595c852ca544aaeeb32d357e62c9c780eac69dcd34e40cae7b55bc4fb1147"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459e81c2fb920b5f5df744262d1498ec2c8081acdcfe18181da44c50f51312f7"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e48cdb8226644e2fbd0bdb0a0f87906a3db07087f4de77a1b1b1ccfd9e93685"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d9b6b28a57feb51605d6ae5e61a9044a31742db557a3b851a74c13bc61de5172"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e594b22688d5747b06e957f1ef822060cb5cb35b493066e33ceac0cf882188b7"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5f236cb5999ccd23a0ab1bd219cfe0ee3e1c1b65aaf6dd3320e972f7ec3a39da"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a2a64e62c7a0edd07c1c917b0586655f3362d2c2d37d474db1a509efb96fea1c"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d0eea830b591dbc68e030c86a9569826145df485b2b4554874b07fea1275a199"}, + {file = "yarl-1.17.1-cp312-cp312-win32.whl", hash = "sha256:46ddf6e0b975cd680eb83318aa1d321cb2bf8d288d50f1754526230fcf59ba96"}, + {file = "yarl-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:117ed8b3732528a1e41af3aa6d4e08483c2f0f2e3d3d7dca7cf538b3516d93df"}, + {file = "yarl-1.17.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5d1d42556b063d579cae59e37a38c61f4402b47d70c29f0ef15cee1acaa64488"}, + {file = "yarl-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0167540094838ee9093ef6cc2c69d0074bbf84a432b4995835e8e5a0d984374"}, + {file = "yarl-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2f0a6423295a0d282d00e8701fe763eeefba8037e984ad5de44aa349002562ac"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5b078134f48552c4d9527db2f7da0b5359abd49393cdf9794017baec7506170"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d401f07261dc5aa36c2e4efc308548f6ae943bfff20fcadb0a07517a26b196d8"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5f1ac7359e17efe0b6e5fec21de34145caef22b260e978336f325d5c84e6938"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f63d176a81555984e91f2c84c2a574a61cab7111cc907e176f0f01538e9ff6e"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e275792097c9f7e80741c36de3b61917aebecc08a67ae62899b074566ff8556"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:81713b70bea5c1386dc2f32a8f0dab4148a2928c7495c808c541ee0aae614d67"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:aa46dce75078fceaf7cecac5817422febb4355fbdda440db55206e3bd288cfb8"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1ce36ded585f45b1e9bb36d0ae94765c6608b43bd2e7f5f88079f7a85c61a4d3"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2d374d70fdc36f5863b84e54775452f68639bc862918602d028f89310a034ab0"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2d9f0606baaec5dd54cb99667fcf85183a7477f3766fbddbe3f385e7fc253299"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b0341e6d9a0c0e3cdc65857ef518bb05b410dbd70d749a0d33ac0f39e81a4258"}, + {file = "yarl-1.17.1-cp313-cp313-win32.whl", hash = "sha256:2e7ba4c9377e48fb7b20dedbd473cbcbc13e72e1826917c185157a137dac9df2"}, + {file = "yarl-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:949681f68e0e3c25377462be4b658500e85ca24323d9619fdc41f68d46a1ffda"}, + {file = "yarl-1.17.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8994b29c462de9a8fce2d591028b986dbbe1b32f3ad600b2d3e1c482c93abad6"}, + {file = "yarl-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f9cbfbc5faca235fbdf531b93aa0f9f005ec7d267d9d738761a4d42b744ea159"}, + {file = "yarl-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b40d1bf6e6f74f7c0a567a9e5e778bbd4699d1d3d2c0fe46f4b717eef9e96b95"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5efe0661b9fcd6246f27957f6ae1c0eb29bc60552820f01e970b4996e016004"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5c4804e4039f487e942c13381e6c27b4b4e66066d94ef1fae3f6ba8b953f383"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5d6a6c9602fd4598fa07e0389e19fe199ae96449008d8304bf5d47cb745462e"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f4c9156c4d1eb490fe374fb294deeb7bc7eaccda50e23775b2354b6a6739934"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6324274b4e0e2fa1b3eccb25997b1c9ed134ff61d296448ab8269f5ac068c4c"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d8a8b74d843c2638f3864a17d97a4acda58e40d3e44b6303b8cc3d3c44ae2d29"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:7fac95714b09da9278a0b52e492466f773cfe37651cf467a83a1b659be24bf71"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c180ac742a083e109c1a18151f4dd8675f32679985a1c750d2ff806796165b55"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:578d00c9b7fccfa1745a44f4eddfdc99d723d157dad26764538fbdda37209857"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1a3b91c44efa29e6c8ef8a9a2b583347998e2ba52c5d8280dbd5919c02dfc3b5"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a7ac5b4984c468ce4f4a553df281450df0a34aefae02e58d77a0847be8d1e11f"}, + {file = "yarl-1.17.1-cp39-cp39-win32.whl", hash = "sha256:7294e38f9aa2e9f05f765b28ffdc5d81378508ce6dadbe93f6d464a8c9594473"}, + {file = "yarl-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:eb6dce402734575e1a8cc0bb1509afca508a400a57ce13d306ea2c663bad1138"}, + {file = "yarl-1.17.1-py3-none-any.whl", hash = "sha256:f1790a4b1e8e8e028c391175433b9c8122c39b46e1663228158e61e6f915bf06"}, + {file = "yarl-1.17.1.tar.gz", hash = "sha256:067a63fcfda82da6b198fa73079b1ca40b7c9b7994995b6ee38acda728b64d47"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" +propcache = ">=0.2.0" + +[[package]] +name = "zipp" +version = "3.21.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.9" +files = [ + {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, + {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] + [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "c8bdf6306d9ba54b5a62aa438d1be669fef3c801ec1c3a6b7b89de1283c61bdd" +content-hash = "80acd51d9be644348cf82ebcd258b260ea43a8a30b5d3025af70f75e5961c5b6" diff --git a/pyproject.toml b/pyproject.toml index 070e9e7..faaf374 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,9 @@ typing-extensions = "^4.12.0" loguru = "^0.7.2" httpx = "^0.27.2" markdownify = "^0.13.1" +litellm = "^1.52.6" +pillow = "^11.0.0" +json-repair = "^0.30.1" [tool.poetry.group.dev.dependencies] From 0e4ec911c286424fef68eae569525c956c69cfc3 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Fri, 29 Nov 2024 11:18:46 +0100 Subject: [PATCH 03/18] working local logic for get_element --- .../browser/_common/_exceptions/_constants.py | 2 +- .../_common/_exceptions/dendrite_exception.py | 1 - dendrite/browser/async_api/__init__.py | 1 + .../browser/async_api/_core/_impl_browser.py | 7 +- .../browser/async_api/_core/_impl_mapping.py | 7 +- .../async_api/_core/_local_browser_impl.py | 6 +- .../_core/_managers/navigation_tracker.py | 1 - .../async_api/_core/_managers/page_manager.py | 7 +- .../browser/async_api/_core/_type_spec.py | 4 +- dendrite/browser/async_api/_core/_utils.py | 20 +- .../async_api/_core/dendrite_browser.py | 114 ++-- .../async_api/_core/dendrite_element.py | 48 +- .../browser/async_api/_core/dendrite_page.py | 44 +- .../browser/async_api/_core/event_sync.py | 6 +- .../browser/async_api/_core/mixin/__init__.py | 24 + dendrite/browser/async_api/_core/mixin/ask.py | 8 +- .../browser/async_api/_core/mixin/click.py | 10 +- .../browser/async_api/_core/mixin/extract.py | 57 +- .../async_api/_core/mixin/fill_fields.py | 7 +- .../async_api/_core/mixin/get_element.py | 34 +- .../browser/async_api/_core/mixin/keyboard.py | 5 +- .../browser/async_api/_core/mixin/markdown.py | 6 +- .../browser/async_api/_core/mixin/wait_for.py | 9 +- .../async_api/_core/models/api_config.py | 33 -- .../async_api/_core/models/authentication.py | 3 +- .../_core/models/download_interface.py | 1 + .../_core/models/page_diff_information.py | 7 - .../_core/models/page_information.py | 5 +- .../async_api/_core/protocol/page_protocol.py | 7 +- .../_remote_impl/browserbase/_client.py | 3 +- .../_remote_impl/browserbase/_download.py | 9 +- .../_remote_impl/browserbase/_impl.py | 12 +- .../_remote_impl/browserless/_impl.py | 15 +- dendrite/browser/remote/__init__.py | 4 +- dendrite/browser/remote/browserbase_config.py | 1 + dendrite/browser/remote/provider.py | 2 - dendrite/browser/sync_api/__init__.py | 7 - dendrite/browser/sync_api/_api/__init__.py | 0 .../browser/sync_api/_api/_http_client.py | 58 -- .../sync_api/_api/browser_api_client.py | 100 ---- .../browser/sync_api/_api/dto/__init__.py | 0 .../browser/sync_api/_api/dto/ask_page_dto.py | 11 - .../sync_api/_api/dto/authenticate_dto.py | 6 - .../browser/sync_api/_api/dto/extract_dto.py | 24 - .../sync_api/_api/dto/get_elements_dto.py | 18 - .../sync_api/_api/dto/get_interaction_dto.py | 9 - .../sync_api/_api/dto/get_session_dto.py | 7 - .../sync_api/_api/dto/google_search_dto.py | 12 - .../sync_api/_api/dto/make_interaction_dto.py | 16 - .../sync_api/_api/dto/try_run_script_dto.py | 12 - .../_api/dto/upload_auth_session_dto.py | 7 - .../sync_api/_api/response/__init__.py | 0 .../_api/response/ask_page_response.py | 10 - .../_api/response/cache_extract_response.py | 5 - .../_api/response/extract_response.py | 13 - .../_api/response/get_element_response.py | 10 - .../_api/response/google_search_response.py | 12 - .../_api/response/selector_cache_response.py | 5 - .../_api/response/session_response.py | 7 - dendrite/browser/sync_api/_common/__init__.py | 0 .../browser/sync_api/_common/constants.py | 66 --- .../browser/sync_api/_common/event_sync.py | 45 -- dendrite/browser/sync_api/_common/status.py | 3 - dendrite/browser/sync_api/_core/__init__.py | 0 .../browser/sync_api/_core/_impl_browser.py | 85 --- .../browser/sync_api/_core/_impl_mapping.py | 28 - .../browser/sync_api/_core/_js/__init__.py | 11 - .../sync_api/_core/_js/eventListenerPatch.js | 90 --- .../sync_api/_core/_js/generateDendriteIDs.js | 85 --- .../_core/_js/generateDendriteIDsIframe.js | 90 --- .../sync_api/_core/_managers/__init__.py | 0 .../_core/_managers/navigation_tracker.py | 67 --- .../sync_api/_core/_managers/page_manager.py | 74 --- .../_core/_managers/screenshot_manager.py | 50 -- dendrite/browser/sync_api/_core/_type_spec.py | 35 -- dendrite/browser/sync_api/_core/_utils.py | 101 ---- .../sync_api/_core/dendrite_browser.py | 428 -------------- .../sync_api/_core/dendrite_element.py | 237 -------- .../browser/sync_api/_core/dendrite_page.py | 375 ------------- dendrite/browser/sync_api/_core/mixin/ask.py | 191 ------- .../browser/sync_api/_core/mixin/click.py | 57 -- .../browser/sync_api/_core/mixin/extract.py | 232 -------- .../sync_api/_core/mixin/fill_fields.py | 76 --- .../sync_api/_core/mixin/get_element.py | 302 ---------- .../browser/sync_api/_core/mixin/keyboard.py | 62 -- .../browser/sync_api/_core/mixin/markdown.py | 23 - .../sync_api/_core/mixin/screenshot.py | 20 - .../browser/sync_api/_core/mixin/wait_for.py | 51 -- .../browser/sync_api/_core/models/__init__.py | 0 .../sync_api/_core/models/api_config.py | 30 - .../sync_api/_core/models/authentication.py | 47 -- .../_core/models/download_interface.py | 20 - .../_core/models/page_diff_information.py | 7 - .../sync_api/_core/models/page_information.py | 15 - .../browser/sync_api/_core/models/response.py | 54 -- .../sync_api/_core/protocol/page_protocol.py | 19 - dendrite/browser/sync_api/_dom/__init__.py | 0 .../browser/sync_api/_ext_impl/__init__.py | 3 - .../_ext_impl/browserbase/__init__.py | 3 - .../sync_api/_ext_impl/browserbase/_client.py | 63 --- .../_ext_impl/browserbase/_download.py | 53 -- .../sync_api/_ext_impl/browserbase/_impl.py | 60 -- .../_ext_impl/browserless/__init__.py | 0 .../sync_api/_ext_impl/browserless/_impl.py | 53 -- dendrite/exceptions/__init__.py | 2 +- dendrite/logic/{local => }/ask/ask.py | 67 +-- dendrite/logic/{local => }/ask/image.py | 4 +- dendrite/logic/cache/element_cache.py | 8 + dendrite/logic/cache/extract_cache.py | 5 + dendrite/logic/cache/file_cache.py | 56 +- dendrite/logic/cache/utils.py | 19 + .../logic/{local => }/code/code_session.py | 31 +- dendrite/logic/config.py | 15 + dendrite/logic/{local => }/dom/css.py | 10 +- dendrite/logic/dom/strip.py | 159 ++++++ dendrite/logic/{local => }/dom/truncate.py | 1 + dendrite/logic/extract/cached_script.py | 42 ++ dendrite/logic/extract/compress_html.py | 491 ++++++++++++++++ dendrite/logic/extract/extract.py | 165 ++++++ dendrite/logic/extract/extract_agent.py | 281 ++++++++++ dendrite/logic/{local => }/extract/prompts.py | 4 +- .../logic/{local => }/extract/scroll_agent.py | 42 +- dendrite/logic/factory.py | 25 +- .../get_element/agents/prompts/__init__.py | 12 + .../get_element/agents/prompts/segment.prompt | 0 .../get_element/agents/prompts/select.prompt | 0 .../get_element/agents/segment_agent.py | 67 +-- .../get_element/agents/select_agent.py | 37 +- dendrite/logic/get_element/cached_selector.py | 35 ++ dendrite/logic/get_element/get_element.py | 112 ++++ .../{local => }/get_element/hanifi_search.py | 77 +-- .../dom.py => get_element/hanifi_segment.py} | 113 +--- .../logic/{local => }/get_element/models.py | 2 +- dendrite/logic/hosted/_api/_http_client.py | 15 +- .../logic/hosted/_api/browser_api_client.py | 96 +--- dendrite/logic/hosted/_api/dto/__init__.py | 0 .../logic/hosted/_api/dto/ask_page_dto.py | 11 - .../logic/hosted/_api/dto/authenticate_dto.py | 6 - dendrite/logic/hosted/_api/dto/extract_dto.py | 25 - .../logic/hosted/_api/dto/get_elements_dto.py | 19 - .../hosted/_api/dto/get_interaction_dto.py | 10 - .../logic/hosted/_api/dto/get_session_dto.py | 7 - .../hosted/_api/dto/google_search_dto.py | 12 - .../hosted/_api/dto/make_interaction_dto.py | 19 - .../hosted/_api/dto/try_run_script_dto.py | 14 - .../_api/dto/upload_auth_session_dto.py | 11 - .../logic/hosted/_api/response/__init__.py | 0 .../_api/response/cache_extract_response.py | 5 - .../_api/response/google_search_response.py | 12 - .../_api/response/interaction_response.py | 7 - .../_api/response/selector_cache_response.py | 5 - .../hosted/_api/response/session_response.py | 7 - dendrite/logic/interfaces/async_api.py | 70 ++- dendrite/logic/interfaces/cache.py | 23 +- dendrite/logic/interfaces/sync_api.py | 75 +++ dendrite/logic/llm/agent.py | 233 ++++++++ dendrite/logic/llm/config.py | 87 +++ dendrite/logic/{local => }/llm/token_count.py | 2 +- dendrite/logic/local/code/execute.py | 26 - dendrite/logic/local/dom/strip.py | 52 -- dendrite/logic/local/extract/extract_agent.py | 529 ------------------ .../logic/local/get_element/agents/agent.py | 146 ----- .../get_element/agents/prompts/__init__.py | 12 - .../local/get_element/cached_selector.py | 56 -- dendrite/logic/local/get_element/main.py | 101 ---- dendrite/logic/verify_interaction.py | 94 ++++ dendrite/models/api_config.py | 5 +- dendrite/models/dto/ask_page_dto.py | 3 +- dendrite/models/dto/extract_dto.py | 23 +- dendrite/models/dto/get_elements_dto.py | 2 +- dendrite/models/dto/make_interaction_dto.py | 19 + dendrite/models/page_information.py | 9 +- .../response/ask_page_response.py | 0 .../models/response/extract_page_response.py | 14 - .../response/extract_response.py | 13 +- .../response/get_element_response.py | 5 +- .../response/interaction_response.py | 3 +- dendrite/models/scripts.py | 7 + dendrite/models/selector.py | 8 + dendrite/models/status.py | 4 + scripts/generate_sync.py | 6 +- 181 files changed, 2413 insertions(+), 5567 deletions(-) create mode 100644 dendrite/browser/async_api/_core/mixin/__init__.py delete mode 100644 dendrite/browser/async_api/_core/models/api_config.py delete mode 100644 dendrite/browser/sync_api/__init__.py delete mode 100644 dendrite/browser/sync_api/_api/__init__.py delete mode 100644 dendrite/browser/sync_api/_api/_http_client.py delete mode 100644 dendrite/browser/sync_api/_api/browser_api_client.py delete mode 100644 dendrite/browser/sync_api/_api/dto/__init__.py delete mode 100644 dendrite/browser/sync_api/_api/dto/ask_page_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/authenticate_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/extract_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/get_elements_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/get_interaction_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/get_session_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/google_search_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/make_interaction_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/try_run_script_dto.py delete mode 100644 dendrite/browser/sync_api/_api/dto/upload_auth_session_dto.py delete mode 100644 dendrite/browser/sync_api/_api/response/__init__.py delete mode 100644 dendrite/browser/sync_api/_api/response/ask_page_response.py delete mode 100644 dendrite/browser/sync_api/_api/response/cache_extract_response.py delete mode 100644 dendrite/browser/sync_api/_api/response/extract_response.py delete mode 100644 dendrite/browser/sync_api/_api/response/get_element_response.py delete mode 100644 dendrite/browser/sync_api/_api/response/google_search_response.py delete mode 100644 dendrite/browser/sync_api/_api/response/selector_cache_response.py delete mode 100644 dendrite/browser/sync_api/_api/response/session_response.py delete mode 100644 dendrite/browser/sync_api/_common/__init__.py delete mode 100644 dendrite/browser/sync_api/_common/constants.py delete mode 100644 dendrite/browser/sync_api/_common/event_sync.py delete mode 100644 dendrite/browser/sync_api/_common/status.py delete mode 100644 dendrite/browser/sync_api/_core/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/_impl_browser.py delete mode 100644 dendrite/browser/sync_api/_core/_impl_mapping.py delete mode 100644 dendrite/browser/sync_api/_core/_js/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/_js/eventListenerPatch.js delete mode 100644 dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js delete mode 100644 dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js delete mode 100644 dendrite/browser/sync_api/_core/_managers/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/_managers/navigation_tracker.py delete mode 100644 dendrite/browser/sync_api/_core/_managers/page_manager.py delete mode 100644 dendrite/browser/sync_api/_core/_managers/screenshot_manager.py delete mode 100644 dendrite/browser/sync_api/_core/_type_spec.py delete mode 100644 dendrite/browser/sync_api/_core/_utils.py delete mode 100644 dendrite/browser/sync_api/_core/dendrite_browser.py delete mode 100644 dendrite/browser/sync_api/_core/dendrite_element.py delete mode 100644 dendrite/browser/sync_api/_core/dendrite_page.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/ask.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/click.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/extract.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/fill_fields.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/get_element.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/keyboard.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/markdown.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/screenshot.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/wait_for.py delete mode 100644 dendrite/browser/sync_api/_core/models/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/models/api_config.py delete mode 100644 dendrite/browser/sync_api/_core/models/authentication.py delete mode 100644 dendrite/browser/sync_api/_core/models/download_interface.py delete mode 100644 dendrite/browser/sync_api/_core/models/page_diff_information.py delete mode 100644 dendrite/browser/sync_api/_core/models/page_information.py delete mode 100644 dendrite/browser/sync_api/_core/models/response.py delete mode 100644 dendrite/browser/sync_api/_core/protocol/page_protocol.py delete mode 100644 dendrite/browser/sync_api/_dom/__init__.py delete mode 100644 dendrite/browser/sync_api/_ext_impl/__init__.py delete mode 100644 dendrite/browser/sync_api/_ext_impl/browserbase/__init__.py delete mode 100644 dendrite/browser/sync_api/_ext_impl/browserbase/_client.py delete mode 100644 dendrite/browser/sync_api/_ext_impl/browserbase/_download.py delete mode 100644 dendrite/browser/sync_api/_ext_impl/browserbase/_impl.py delete mode 100644 dendrite/browser/sync_api/_ext_impl/browserless/__init__.py delete mode 100644 dendrite/browser/sync_api/_ext_impl/browserless/_impl.py rename dendrite/logic/{local => }/ask/ask.py (84%) rename dendrite/logic/{local => }/ask/image.py (100%) create mode 100644 dendrite/logic/cache/extract_cache.py create mode 100644 dendrite/logic/cache/utils.py rename dendrite/logic/{local => }/code/code_session.py (87%) create mode 100644 dendrite/logic/config.py rename dendrite/logic/{local => }/dom/css.py (98%) create mode 100644 dendrite/logic/dom/strip.py rename dendrite/logic/{local => }/dom/truncate.py (99%) create mode 100644 dendrite/logic/extract/cached_script.py create mode 100644 dendrite/logic/extract/compress_html.py create mode 100644 dendrite/logic/extract/extract.py create mode 100644 dendrite/logic/extract/extract_agent.py rename dendrite/logic/{local => }/extract/prompts.py (99%) rename dendrite/logic/{local => }/extract/scroll_agent.py (89%) create mode 100644 dendrite/logic/get_element/agents/prompts/__init__.py rename dendrite/logic/{local => }/get_element/agents/prompts/segment.prompt (100%) rename dendrite/logic/{local => }/get_element/agents/prompts/select.prompt (100%) rename dendrite/logic/{local => }/get_element/agents/segment_agent.py (70%) rename dendrite/logic/{local => }/get_element/agents/select_agent.py (79%) create mode 100644 dendrite/logic/get_element/cached_selector.py create mode 100644 dendrite/logic/get_element/get_element.py rename dendrite/logic/{local => }/get_element/hanifi_search.py (69%) rename dendrite/logic/{local/get_element/dom.py => get_element/hanifi_segment.py} (75%) rename dendrite/logic/{local => }/get_element/models.py (93%) delete mode 100644 dendrite/logic/hosted/_api/dto/__init__.py delete mode 100644 dendrite/logic/hosted/_api/dto/ask_page_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/authenticate_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/extract_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/get_elements_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/get_interaction_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/get_session_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/google_search_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/make_interaction_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/try_run_script_dto.py delete mode 100644 dendrite/logic/hosted/_api/dto/upload_auth_session_dto.py delete mode 100644 dendrite/logic/hosted/_api/response/__init__.py delete mode 100644 dendrite/logic/hosted/_api/response/cache_extract_response.py delete mode 100644 dendrite/logic/hosted/_api/response/google_search_response.py delete mode 100644 dendrite/logic/hosted/_api/response/interaction_response.py delete mode 100644 dendrite/logic/hosted/_api/response/selector_cache_response.py delete mode 100644 dendrite/logic/hosted/_api/response/session_response.py create mode 100644 dendrite/logic/interfaces/sync_api.py create mode 100644 dendrite/logic/llm/agent.py create mode 100644 dendrite/logic/llm/config.py rename dendrite/logic/{local => }/llm/token_count.py (67%) delete mode 100644 dendrite/logic/local/code/execute.py delete mode 100644 dendrite/logic/local/dom/strip.py delete mode 100644 dendrite/logic/local/extract/extract_agent.py delete mode 100644 dendrite/logic/local/get_element/agents/agent.py delete mode 100644 dendrite/logic/local/get_element/agents/prompts/__init__.py delete mode 100644 dendrite/logic/local/get_element/cached_selector.py delete mode 100644 dendrite/logic/local/get_element/main.py create mode 100644 dendrite/logic/verify_interaction.py create mode 100644 dendrite/models/dto/make_interaction_dto.py rename dendrite/{logic/hosted/_api => models}/response/ask_page_response.py (100%) delete mode 100644 dendrite/models/response/extract_page_response.py rename dendrite/{logic/hosted/_api => models}/response/extract_response.py (65%) rename dendrite/{logic/hosted/_api => models}/response/get_element_response.py (77%) rename dendrite/{browser/sync_api/_api => models}/response/interaction_response.py (64%) create mode 100644 dendrite/models/scripts.py create mode 100644 dendrite/models/selector.py create mode 100644 dendrite/models/status.py diff --git a/dendrite/browser/_common/_exceptions/_constants.py b/dendrite/browser/_common/_exceptions/_constants.py index a507e87..f970308 100644 --- a/dendrite/browser/_common/_exceptions/_constants.py +++ b/dendrite/browser/_common/_exceptions/_constants.py @@ -1 +1 @@ -INVALID_AUTH_SESSION_MSG = "Missing auth session for any of: {domain}. Make sure that you have used the Dendrite Vault extension to extract your authenticated session(s) for the domain(s) you are trying to access." \ No newline at end of file +INVALID_AUTH_SESSION_MSG = "Missing auth session for any of: {domain}. Make sure that you have used the Dendrite Vault extension to extract your authenticated session(s) for the domain(s) you are trying to access." diff --git a/dendrite/browser/_common/_exceptions/dendrite_exception.py b/dendrite/browser/_common/_exceptions/dendrite_exception.py index 1eaa160..1b2b86b 100644 --- a/dendrite/browser/_common/_exceptions/dendrite_exception.py +++ b/dendrite/browser/_common/_exceptions/dendrite_exception.py @@ -110,7 +110,6 @@ class IncorrectOutcomeError(BaseDendriteException): Inherits from BaseDendriteException. """ - pass class BrowserNotLaunchedError(BaseDendriteException): diff --git a/dendrite/browser/async_api/__init__.py b/dendrite/browser/async_api/__init__.py index 48accf0..eb4bd3c 100644 --- a/dendrite/browser/async_api/__init__.py +++ b/dendrite/browser/async_api/__init__.py @@ -1,4 +1,5 @@ from loguru import logger + from ._core.dendrite_browser import AsyncDendrite from ._core.dendrite_element import AsyncElement from ._core.dendrite_page import AsyncPage diff --git a/dendrite/browser/async_api/_core/_impl_browser.py b/dendrite/browser/async_api/_core/_impl_browser.py index 97709a4..6754927 100644 --- a/dendrite/browser/async_api/_core/_impl_browser.py +++ b/dendrite/browser/async_api/_core/_impl_browser.py @@ -4,8 +4,9 @@ if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite +from playwright.async_api import Browser, Download, Playwright + from dendrite.browser.async_api._core._type_spec import PlaywrightPage -from playwright.async_api import Download, Browser, Playwright class ImplBrowser(ABC): @@ -27,7 +28,6 @@ async def get_download( Raises: Exception: If there is an issue retrieving the download event. """ - pass @abstractmethod async def start_browser(self, playwright: Playwright, pw_options: dict) -> Browser: @@ -40,7 +40,6 @@ async def start_browser(self, playwright: Playwright, pw_options: dict) -> Brows Raises: Exception: If there is an issue starting the browser session. """ - pass @abstractmethod async def configure_context(self, browser: "AsyncDendrite") -> None: @@ -53,7 +52,6 @@ async def configure_context(self, browser: "AsyncDendrite") -> None: Raises: Exception: If there is an issue configuring the browser context. """ - pass @abstractmethod async def stop_session(self) -> None: @@ -63,4 +61,3 @@ async def stop_session(self) -> None: Raises: Exception: If there is an issue stopping the browser session. """ - pass \ No newline at end of file diff --git a/dendrite/browser/async_api/_core/_impl_mapping.py b/dendrite/browser/async_api/_core/_impl_mapping.py index 7022aa5..60bccfa 100644 --- a/dendrite/browser/async_api/_core/_impl_mapping.py +++ b/dendrite/browser/async_api/_core/_impl_mapping.py @@ -1,13 +1,12 @@ -from typing import Any, Dict, Optional, Type +from typing import Dict, Optional, Type from dendrite.browser.async_api._core._impl_browser import ImplBrowser - from dendrite.browser.async_api._core._local_browser_impl import LocalImpl from dendrite.browser.async_api._remote_impl.browserbase._impl import BrowserBaseImpl from dendrite.browser.async_api._remote_impl.browserless._impl import BrowserlessImpl -from dendrite.browser.remote.browserless_config import BrowserlessConfig -from dendrite.browser.remote.browserbase_config import BrowserbaseConfig from dendrite.browser.remote import Providers +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser.remote.browserless_config import BrowserlessConfig IMPL_MAPPING: Dict[Type[Providers], Type[ImplBrowser]] = { BrowserbaseConfig: BrowserBaseImpl, diff --git a/dendrite/browser/async_api/_core/_local_browser_impl.py b/dendrite/browser/async_api/_core/_local_browser_impl.py index 61c0256..a524ae6 100644 --- a/dendrite/browser/async_api/_core/_local_browser_impl.py +++ b/dendrite/browser/async_api/_core/_local_browser_impl.py @@ -3,9 +3,11 @@ if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite +from playwright.async_api import Browser, Download, Playwright + from dendrite.browser.async_api._core._impl_browser import ImplBrowser from dendrite.browser.async_api._core._type_spec import PlaywrightPage -from playwright.async_api import Download, Browser, Playwright + class LocalImpl(ImplBrowser): def __init__(self) -> None: @@ -26,4 +28,4 @@ async def configure_context(self, browser: "AsyncDendrite"): pass async def stop_session(self): - pass \ No newline at end of file + pass diff --git a/dendrite/browser/async_api/_core/_managers/navigation_tracker.py b/dendrite/browser/async_api/_core/_managers/navigation_tracker.py index 71a1f05..ac9b578 100644 --- a/dendrite/browser/async_api/_core/_managers/navigation_tracker.py +++ b/dendrite/browser/async_api/_core/_managers/navigation_tracker.py @@ -1,6 +1,5 @@ import asyncio import time - from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: diff --git a/dendrite/browser/async_api/_core/_managers/page_manager.py b/dendrite/browser/async_api/_core/_managers/page_manager.py index 76e5b01..34bf20b 100644 --- a/dendrite/browser/async_api/_core/_managers/page_manager.py +++ b/dendrite/browser/async_api/_core/_managers/page_manager.py @@ -1,10 +1,11 @@ -from typing import Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from loguru import logger from playwright.async_api import BrowserContext, Download, FileChooser if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from dendrite.browser.async_api._core._type_spec import PlaywrightPage from dendrite.browser.async_api._core.dendrite_page import AsyncPage @@ -25,7 +26,7 @@ async def new_page(self) -> AsyncPage: if self.active_page and new_page == self.active_page.playwright_page: return self.active_page - client = self.dendrite_browser._get_browser_api_client() + client = self.dendrite_browser._get_logic_api() dendrite_page = AsyncPage(new_page, self.dendrite_browser, client) self.pages.append(dendrite_page) self.active_page = dendrite_page @@ -75,7 +76,7 @@ def _page_on_open_handler(self, page: PlaywrightPage): page.on("download", self._page_on_download_handler) page.on("filechooser", self._page_on_filechooser_handler) - client = self.dendrite_browser._get_browser_api_client() + client = self.dendrite_browser._get_logic_api() dendrite_page = AsyncPage(page, self.dendrite_browser, client) self.pages.append(dendrite_page) self.active_page = dendrite_page diff --git a/dendrite/browser/async_api/_core/_type_spec.py b/dendrite/browser/async_api/_core/_type_spec.py index 8252e08..872e1c4 100644 --- a/dendrite/browser/async_api/_core/_type_spec.py +++ b/dendrite/browser/async_api/_core/_type_spec.py @@ -1,8 +1,8 @@ import inspect from typing import Any, Dict, Literal, Type, TypeVar, Union -from pydantic import BaseModel -from playwright.async_api import Page +from playwright.async_api import Page +from pydantic import BaseModel Interaction = Literal["click", "fill", "hover"] diff --git a/dendrite/browser/async_api/_core/_utils.py b/dendrite/browser/async_api/_core/_utils.py index 6fd7142..4e91bf1 100644 --- a/dendrite/browser/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_core/_utils.py @@ -1,20 +1,19 @@ -from typing import Optional, Union, List, TYPE_CHECKING -from playwright.async_api import FrameLocator, ElementHandle, Error, Frame +from typing import TYPE_CHECKING, List, Optional, Union + from bs4 import BeautifulSoup from loguru import logger +from playwright.async_api import ElementHandle, Error, Frame, FrameLocator -from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse from dendrite.browser.async_api._core._type_spec import PlaywrightPage from dendrite.browser.async_api._core.dendrite_element import AsyncElement from dendrite.browser.async_api._core.models.response import AsyncElementsResponse +from dendrite.models.response.get_element_response import GetElementResponse if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_page import AsyncPage -from dendrite.browser.async_api._core._js import ( - GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, -) -from dendrite.browser.async_api._dom.mild_strip import mild_strip_in_place +from dendrite.browser.async_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT +from dendrite.logic.dom.strip import mild_strip_in_place async def expand_iframes( @@ -36,8 +35,11 @@ async def get_iframe_path(frame: Frame): for frame in page.frames: if frame.parent_frame is None: - continue # Skip the main frame - iframe_element = await frame.frame_element() + continue # Skip the main frame + try: + iframe_element = await frame.frame_element() + except Error as e: + continue iframe_id = await iframe_element.get_attribute("d-id") if iframe_id is None: continue diff --git a/dendrite/browser/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/_core/dendrite_browser.py index 9abec39..a9706c5 100644 --- a/dendrite/browser/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/_core/dendrite_browser.py @@ -1,54 +1,47 @@ -from abc import ABC, abstractmethod +import os import pathlib import re -from typing import Any, List, Literal, Optional, Sequence, Union +from abc import ABC +from typing import Any, List, Optional, Sequence, Union from uuid import uuid4 -import os + from loguru import logger from playwright.async_api import ( - async_playwright, - Playwright, BrowserContext, - FileChooser, Download, Error, + FileChooser, FilePayload, + Playwright, + async_playwright, ) -from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.browser.async_api._core.event_sync import EventSync -from dendrite.browser.async_api._core._impl_browser import ImplBrowser -from dendrite.browser.async_api._core._impl_mapping import get_impl -from dendrite.browser.async_api._core._managers.page_manager import ( - PageManager, -) - -from dendrite.browser.async_api._core._type_spec import PlaywrightPage -from dendrite.browser.async_api._core.dendrite_page import AsyncPage -from dendrite.browser.async_api._common.constants import STEALTH_ARGS -from dendrite.browser.async_api._core.mixin.ask import AskMixin -from dendrite.browser.async_api._core.mixin.click import ClickMixin -from dendrite.browser.async_api._core.mixin.extract import ExtractionMixin -from dendrite.browser.async_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.async_api._core.mixin.keyboard import KeyboardMixin -from dendrite.browser.async_api._core.mixin.screenshot import ScreenshotMixin -from dendrite.browser.async_api._core.mixin.wait_for import WaitForMixin -from dendrite.browser.async_api._core.mixin.markdown import MarkdownMixin -from dendrite.browser.async_api._core.models.authentication import ( - AuthSession, -) - -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._api.browser_api_client import BrowserAPIClient from dendrite.browser._common._exceptions.dendrite_exception import ( BrowserNotLaunchedError, DendriteException, IncorrectOutcomeError, ) +from dendrite.browser._common.constants import STEALTH_ARGS +from dendrite.models.api_config import APIConfig +from dendrite.browser.async_api._core._impl_browser import ImplBrowser +from dendrite.browser.async_api._core._impl_mapping import get_impl +from dendrite.browser.async_api._core._managers.page_manager import PageManager +from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api._core.dendrite_page import AsyncPage +from dendrite.browser.async_api._core.event_sync import EventSync +from dendrite.browser.async_api._core.mixin import ( + AskMixin, + ClickMixin, + ExtractionMixin, + FillFieldsMixin, + GetElementMixin, + KeyboardMixin, + MarkdownMixin, + ScreenshotMixin, + WaitForMixin, + ) from dendrite.browser.remote import Providers -from dendrite.logic.interfaces.async_api import BrowserAPIProtocol +from dendrite.logic.interfaces.async_api import LocalProtocol, LogicAPIProtocol class AsyncDendrite( @@ -88,7 +81,6 @@ class AsyncDendrite( def __init__( self, - auth: Optional[Union[str, List[str]]] = None, dendrite_api_key: Optional[str] = None, openai_api_key: Optional[str] = None, anthropic_api_key: Optional[str] = None, @@ -102,7 +94,6 @@ def __init__( Initializes AsyncDendrite with API keys and Playwright options. Args: - auth (Optional[Union[str, List[str]]]): The domains on which the browser should try and authenticate. dendrite_api_key (Optional[str]): The Dendrite API key. If not provided, it's fetched from the environment variables. openai_api_key (Optional[str]): Your own OpenAI API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. anthropic_api_key (Optional[str]): The own Anthropic API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. @@ -112,15 +103,15 @@ def __init__( MissingApiKeyError: If the Dendrite API key is not provided or found in the environment variables. """ - api_config = APIConfig( - dendrite_api_key=dendrite_api_key or os.environ.get("DENDRITE_API_KEY"), - openai_api_key=openai_api_key, - anthropic_api_key=anthropic_api_key, - ) + # api_config = APIConfig( + # dendrite_api_key=dendrite_api_key or os.environ.get("DENDRITE_API_KEY"), + # openai_api_key=openai_api_key, + # anthropic_api_key=anthropic_api_key, + # ) self._impl = self._get_impl(remote_config) - self.api_config = api_config + # self.api_config = api_config self.playwright: Optional[Playwright] = None self.browser_context: Optional[BrowserContext] = None @@ -131,8 +122,7 @@ def __init__( self._upload_handler = EventSync(event_type=FileChooser) self._download_handler = EventSync(event_type=Download) self.closed = False - self._auth = auth - self._browser_api_client: BrowserAPIProtocol = BrowserAPIClient(api_config, self._id) + self._browser_api_client: LogicAPIProtocol = LocalProtocol() @property def pages(self) -> List[AsyncPage]: @@ -151,7 +141,7 @@ async def _get_page(self) -> AsyncPage: active_page = await self.get_active_page() return active_page - def _get_browser_api_client(self) -> BrowserAPIProtocol: + def _get_logic_api(self) -> 'LogicAPIProtocol': return self._browser_api_client def _get_dendrite_browser(self) -> "AsyncDendrite": @@ -168,11 +158,6 @@ def _get_impl(self, remote_provider: Optional[Providers]) -> ImplBrowser: # if remote_provider is None:) return get_impl(remote_provider) - async def _get_auth_session(self, domains: Union[str, list[str]]): - dto = AuthenticateDTO(domains=domains) - auth_session: AuthSession = await self._browser_api_client.authenticate(dto) - return auth_session - async def get_active_page(self) -> AsyncPage: """ Retrieves the currently active page managed by the PageManager. @@ -301,18 +286,13 @@ async def _launch(self): self._playwright, self._playwright_options ) - if self._auth: - auth_session = await self._get_auth_session(self._auth) - self.browser_context = await browser.new_context( - storage_state=auth_session.to_storage_state(), - user_agent=auth_session.user_agent, - ) - else: - self.browser_context = ( - browser.contexts[0] - if len(browser.contexts) > 0 - else await browser.new_context() - ) + + + self.browser_context = ( + browser.contexts[0] + if len(browser.contexts) > 0 + else await browser.new_context() + ) self._active_page_manager = PageManager(self, self.browser_context) @@ -339,8 +319,7 @@ async def close(self): """ Closes the browser and uploads authentication session data if available. - This method stops the Playwright instance, closes the browser context, and uploads any - stored authentication session data if applicable. + This method stops the Playwright instance, closes the browser context Returns: None @@ -352,13 +331,6 @@ async def close(self): self.closed = True try: if self.browser_context: - if self._auth: - auth_session = await self._get_auth_session(self._auth) - storage_state = await self.browser_context.storage_state() - dto = UploadAuthSessionDTO( - auth_data=auth_session, storage_state=storage_state - ) - await self._browser_api_client.upload_auth_session(dto) await self._impl.stop_session() await self.browser_context.close() except Error: diff --git a/dendrite/browser/async_api/_core/dendrite_element.py b/dendrite/browser/async_api/_core/dendrite_element.py index 6c44ecf..16dfaec 100644 --- a/dendrite/browser/async_api/_core/dendrite_element.py +++ b/dendrite/browser/async_api/_core/dendrite_element.py @@ -1,4 +1,5 @@ from __future__ import annotations + import asyncio import base64 import functools @@ -8,21 +9,21 @@ from loguru import logger from playwright.async_api import Locator -from dendrite.browser.async_api._api.browser_api_client import BrowserAPIClient -from dendrite.browser._common._exceptions.dendrite_exception import IncorrectOutcomeError -from dendrite.logic.interfaces.async_api import BrowserAPIProtocol +from dendrite.browser._common._exceptions.dendrite_exception import ( + IncorrectOutcomeError, +) + +from dendrite.logic.interfaces.async_api import LogicAPIProtocol if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.browser.async_api._core._managers.navigation_tracker import NavigationTracker -from dendrite.browser.async_api._core.models.page_diff_information import ( - PageDiffInformation, + +from dendrite.browser.async_api._core._managers.navigation_tracker import ( + NavigationTracker, ) from dendrite.browser.async_api._core._type_spec import Interaction -from dendrite.browser.async_api._api.response.interaction_response import ( - InteractionResponse, -) -from dendrite.browser.async_api._api.dto.make_interaction_dto import MakeInteractionDTO +from dendrite.models.dto.make_interaction_dto import VerifyActionDTO +from dendrite.models.response.interaction_response import InteractionResponse def perform_action(interaction_type: Interaction): @@ -52,11 +53,11 @@ async def wrapper( await func(self, *args, **kwargs) return InteractionResponse(status="success", message="") - api_config = self._dendrite_browser.api_config - page_before = await self._dendrite_browser.get_active_page() page_before_info = await page_before.get_page_information() - + soup = await page_before._get_previous_soup() + screenshot_before = page_before_info.screenshot_base64 + tag_name = soup.find(attrs={"d-id": self.dendrite_id}) # Call the original method here await func( self, @@ -64,29 +65,28 @@ async def wrapper( *args, **kwargs, ) - + await self._wait_for_page_changes(page_before.url) page_after = await self._dendrite_browser.get_active_page() - page_after_info = await page_after.get_page_information() - page_delta_information = PageDiffInformation( - page_before=page_before_info, page_after=page_after_info - ) + screenshot_after = await page_after.screenshot_manager.take_full_page_screenshot() + - dto = MakeInteractionDTO( + dto = VerifyActionDTO( url=page_before.url, dendrite_id=self.dendrite_id, interaction_type=interaction_type, expected_outcome=expected_outcome, - page_delta_information=page_delta_information, - api_config=api_config, + screenshot_before=screenshot_before, + screenshot_after=screenshot_after, + tag_name = str(tag_name), ) - res = await self._browser_api_client.make_interaction(dto) + res = await self._browser_api_client.verify_action(dto) if res.status == "failed": raise IncorrectOutcomeError( message=res.message, - screenshot_base64=page_delta_information.page_after.screenshot_base64, + screenshot_base64=screenshot_after ) return res @@ -109,7 +109,7 @@ def __init__( dendrite_id: str, locator: Locator, dendrite_browser: AsyncDendrite, - browser_api_client: BrowserAPIProtocol, + browser_api_client: LogicAPIProtocol, ): """ Initialize a AsyncElement. diff --git a/dendrite/browser/async_api/_core/dendrite_page.py b/dendrite/browser/async_api/_core/dendrite_page.py index 13c5db3..70190c3 100644 --- a/dendrite/browser/async_api/_core/dendrite_page.py +++ b/dendrite/browser/async_api/_core/dendrite_page.py @@ -1,30 +1,13 @@ -import re import asyncio import pathlib +import re import time - -from typing import ( - TYPE_CHECKING, - Any, - List, - Literal, - Optional, - Sequence, - Union, -) +from typing import TYPE_CHECKING, Any, List, Literal, Optional, Sequence, Union from bs4 import BeautifulSoup, Tag from loguru import logger +from playwright.async_api import Download, FilePayload, FrameLocator, Keyboard -from playwright.async_api import ( - FrameLocator, - Keyboard, - Download, - FilePayload, -) - - -from dendrite.browser.async_api._api.browser_api_client import BrowserAPIClient from dendrite.browser.async_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT from dendrite.browser.async_api._core._type_spec import PlaywrightPage from dendrite.browser.async_api._core.dendrite_element import AsyncElement @@ -36,23 +19,18 @@ from dendrite.browser.async_api._core.mixin.keyboard import KeyboardMixin from dendrite.browser.async_api._core.mixin.markdown import MarkdownMixin from dendrite.browser.async_api._core.mixin.wait_for import WaitForMixin -from dendrite.browser.async_api._core.models.page_information import PageInformation -from dendrite.logic.interfaces.async_api import BrowserAPIProtocol +from dendrite.logic.interfaces.async_api import LogicAPIProtocol +from dendrite.models.page_information import PageInformation if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite - -from dendrite.browser.async_api._core._managers.screenshot_manager import ScreenshotManager -from dendrite.browser._common._exceptions.dendrite_exception import ( - DendriteException, -) - - -from dendrite.browser.async_api._core._utils import ( - expand_iframes, +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.async_api._core._managers.screenshot_manager import ( + ScreenshotManager, ) +from dendrite.browser.async_api._core._utils import expand_iframes class AsyncPage( @@ -76,7 +54,7 @@ def __init__( self, page: PlaywrightPage, dendrite_browser: "AsyncDendrite", - browser_api_client: "BrowserAPIProtocol", + browser_api_client: "LogicAPIProtocol", ): self.playwright_page = page self.screenshot_manager = ScreenshotManager(page) @@ -118,7 +96,7 @@ async def _get_page(self) -> "AsyncPage": def _get_dendrite_browser(self) -> "AsyncDendrite": return self.dendrite_browser - def _get_browser_api_client(self) -> BrowserAPIProtocol: + def _get_logic_api(self) -> LogicAPIProtocol: return self._browser_api_client async def goto( diff --git a/dendrite/browser/async_api/_core/event_sync.py b/dendrite/browser/async_api/_core/event_sync.py index db93358..a953aee 100644 --- a/dendrite/browser/async_api/_core/event_sync.py +++ b/dendrite/browser/async_api/_core/event_sync.py @@ -1,8 +1,8 @@ -import time import asyncio -from typing import Generic, Optional, Type, TypeVar, Union, cast -from playwright.async_api import Page, Download, FileChooser +import time +from typing import Generic, Optional, Type, TypeVar +from playwright.async_api import Download, FileChooser, Page Events = TypeVar("Events", Download, FileChooser) diff --git a/dendrite/browser/async_api/_core/mixin/__init__.py b/dendrite/browser/async_api/_core/mixin/__init__.py new file mode 100644 index 0000000..140ba4a --- /dev/null +++ b/dendrite/browser/async_api/_core/mixin/__init__.py @@ -0,0 +1,24 @@ + +from .ask import AskMixin +from .click import ClickMixin +from .extract import ExtractionMixin +from .fill_fields import FillFieldsMixin +from .get_element import GetElementMixin +from .keyboard import KeyboardMixin +from .markdown import MarkdownMixin +from .screenshot import ScreenshotMixin +from .wait_for import WaitForMixin + + +__all__ = [ + "AskMixin", + "ClickMixin", + "ExtractionMixin", + "FillFieldsMixin", + "GetElementMixin", + "KeyboardMixin", + "MarkdownMixin", + "ScreenshotMixin", + "WaitForMixin", +] + diff --git a/dendrite/browser/async_api/_core/mixin/ask.py b/dendrite/browser/async_api/_core/mixin/ask.py index 79f9401..5a4c217 100644 --- a/dendrite/browser/async_api/_core/mixin/ask.py +++ b/dendrite/browser/async_api/_core/mixin/ask.py @@ -4,7 +4,7 @@ from loguru import logger -from dendrite.browser.async_api._api.dto.ask_page_dto import AskPageDTO +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException from dendrite.browser.async_api._core._type_spec import ( JsonSchema, PydanticModel, @@ -13,7 +13,7 @@ to_json_schema, ) from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.models.dto.ask_page_dto import AskPageDTO # The timeout interval between retries in milliseconds TIMEOUT_INTERVAL = [150, 450, 1000] @@ -135,7 +135,6 @@ async def ask( Raises: DendriteException: If the request fails, the exception includes the failure message and a screenshot. """ - api_config = self._get_dendrite_browser().api_config start_time = time.time() attempt_start = start_time attempt = -1 @@ -182,13 +181,12 @@ async def ask( dto = AskPageDTO( page_information=page_information, - api_config=api_config, prompt=entire_prompt, return_schema=schema, ) try: - res = await self._get_browser_api_client().ask_page(dto) + res = await self._get_logic_api().ask_page(dto) logger.debug(f"Got response in {time.time() - attempt_start} seconds") if res.status == "error": diff --git a/dendrite/browser/async_api/_core/mixin/click.py b/dendrite/browser/async_api/_core/mixin/click.py index 8dac206..1b3b110 100644 --- a/dendrite/browser/async_api/_core/mixin/click.py +++ b/dendrite/browser/async_api/_core/mixin/click.py @@ -1,11 +1,9 @@ -import asyncio -from typing import Any, Optional -from dendrite.browser.async_api._api.response.interaction_response import ( - InteractionResponse, -) +from typing import Optional + +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.models.response.interaction_response import InteractionResponse class ClickMixin(GetElementMixin, DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/mixin/extract.py b/dendrite/browser/async_api/_core/mixin/extract.py index 55d756c..f72dfae 100644 --- a/dendrite/browser/async_api/_core/mixin/extract.py +++ b/dendrite/browser/async_api/_core/mixin/extract.py @@ -1,11 +1,12 @@ import asyncio import time -from typing import Any, Optional, Type, overload, List -from dendrite.browser.async_api._api.dto.extract_dto import ExtractDTO -from dendrite.browser.async_api._api.response.cache_extract_response import ( - CacheExtractResponse, +from typing import Any, List, Optional, Type, overload + +from loguru import logger + +from dendrite.browser.async_api._core._managers.navigation_tracker import ( + NavigationTracker, ) -from dendrite.browser.async_api._api.response.extract_response import ExtractResponse from dendrite.browser.async_api._core._type_spec import ( JsonSchema, PydanticModel, @@ -14,9 +15,8 @@ to_json_schema, ) from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser.async_api._core._managers.navigation_tracker import NavigationTracker -from loguru import logger - +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.response.extract_response import ExtractResponse CACHE_TIMEOUT = 5 @@ -133,21 +133,16 @@ async def extract( # Check if a script exists in the cache if use_cache: - cache_available = await check_if_extract_cache_available( - self, prompt, json_schema + logger.info("Cache available, attempting to use cached extraction") + result = await attempt_extraction_with_backoff( + self, + prompt, + json_schema, + remaining_timeout=CACHE_TIMEOUT, + only_use_cache=True, ) - - if cache_available: - logger.info("Cache available, attempting to use cached extraction") - result = await attempt_extraction_with_backoff( - self, - prompt, - json_schema, - remaining_timeout=CACHE_TIMEOUT, - only_use_cache=True, - ) - if result: - return convert_and_return_result(result, type_spec) + if result: + return convert_and_return_result(result, type_spec) logger.info( "Using extraction agent to perform extraction, since no cache was found or failed." @@ -167,21 +162,6 @@ async def extract( return None -async def check_if_extract_cache_available( - obj: DendritePageProtocol, prompt: str, json_schema: Optional[JsonSchema] -) -> bool: - page = await obj._get_page() - page_information = await page.get_page_information(include_screenshot=False) - dto = ExtractDTO( - page_information=page_information, - api_config=obj._get_dendrite_browser().api_config, - prompt=prompt, - return_data_json_schema=json_schema, - ) - cache_response: CacheExtractResponse = ( - await obj._get_browser_api_client().check_extract_cache(dto) - ) - return cache_response.exists async def attempt_extraction_with_backoff( @@ -207,7 +187,6 @@ async def attempt_extraction_with_backoff( ) extract_dto = ExtractDTO( page_information=page_information, - api_config=obj._get_dendrite_browser().api_config, prompt=prompt, return_data_json_schema=json_schema, use_screenshot=True, @@ -215,7 +194,7 @@ async def attempt_extraction_with_backoff( force_use_cache=only_use_cache, ) - res = await obj._get_browser_api_client().extract(extract_dto) + res: ExtractResponse = await obj._get_logic_api().extract(extract_dto) request_duration = time.time() - request_start_time if res.status == "impossible": diff --git a/dendrite/browser/async_api/_core/mixin/fill_fields.py b/dendrite/browser/async_api/_core/mixin/fill_fields.py index 92ed5a4..cb2701a 100644 --- a/dendrite/browser/async_api/_core/mixin/fill_fields.py +++ b/dendrite/browser/async_api/_core/mixin/fill_fields.py @@ -1,11 +1,10 @@ import asyncio from typing import Any, Dict, Optional -from dendrite.browser.async_api._api.response.interaction_response import ( - InteractionResponse, -) + +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.models.response.interaction_response import InteractionResponse class FillFieldsMixin(GetElementMixin, DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/mixin/get_element.py b/dendrite/browser/async_api/_core/mixin/get_element.py index fd5da2a..3ba39eb 100644 --- a/dendrite/browser/async_api/_core/mixin/get_element.py +++ b/dendrite/browser/async_api/_core/mixin/get_element.py @@ -4,15 +4,12 @@ from loguru import logger -from dendrite.browser.async_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse -from dendrite.browser.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO from dendrite.browser.async_api._core._utils import get_elements_from_selectors_soup from dendrite.browser.async_api._core.dendrite_element import AsyncElement from dendrite.browser.async_api._core.models.response import AsyncElementsResponse from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser.async_api._core.models.api_config import APIConfig - +from dendrite.models.dto.get_elements_dto import CheckSelectorCacheDTO, GetElementsDTO +from dendrite.models.response.get_element_response import GetElementResponse CACHE_TIMEOUT = 5 @@ -199,23 +196,20 @@ async def _get_element( Union[AsyncElement, List[AsyncElement], AsyncElementsResponse]: The retrieved element, list of elements, or response object. """ - api_config = self._get_dendrite_browser().api_config + start_time = time.time() # First, let's check if there is a cached selector page = await self._get_page() - cache_available = await test_if_cache_available( - self, prompt_or_elements, page.url - ) + # If we have cached elements, attempt to use them with an exponentation backoff - if cache_available and use_cache == True: - logger.info(f"Cache available, attempting to use cached selectors") + if use_cache == True: + logger.debug("Attempting to use cached selectors") res = await attempt_with_backoff( self, prompt_or_elements, only_one, - api_config, remaining_timeout=CACHE_TIMEOUT, only_use_cache=True, ) @@ -234,7 +228,6 @@ async def _get_element( self, prompt_or_elements, only_one, - api_config, remaining_timeout=timeout - (time.time() - start_time), only_use_cache=False, ) @@ -247,23 +240,12 @@ async def _get_element( return None -async def test_if_cache_available( - obj: DendritePageProtocol, prompt_or_elements: Union[str, Dict[str, str]], url: str -) -> bool: - dto = CheckSelectorCacheDTO( - url=url, - prompt=prompt_or_elements, - ) - cache_available = await obj._get_browser_api_client().check_selector_cache(dto) - - return cache_available.exists async def attempt_with_backoff( obj: DendritePageProtocol, prompt_or_elements: Union[str, Dict[str, str]], only_one: bool, - api_config: APIConfig, remaining_timeout: float, only_use_cache: bool = False, ) -> Union[Optional[AsyncElement], List[AsyncElement], AsyncElementsResponse]: @@ -284,12 +266,11 @@ async def attempt_with_backoff( dto = GetElementsDTO( page_information=page_information, prompt=prompt_or_elements, - api_config=api_config, use_cache=only_use_cache, only_one=only_one, force_use_cache=only_use_cache, ) - res = await obj._get_browser_api_client().get_interactions_selector(dto) + res = await obj._get_logic_api().get_element(dto) request_duration = time.time() - request_start_time if res.status == "impossible": @@ -299,6 +280,7 @@ async def attempt_with_backoff( return None if res.status == "success": + logger.success(f"d[id]: {res.d_id} Selectors:{res.selectors}") response = await get_elements_from_selectors_soup( page, await page._get_previous_soup(), res, only_one ) diff --git a/dendrite/browser/async_api/_core/mixin/keyboard.py b/dendrite/browser/async_api/_core/mixin/keyboard.py index 8250370..008c464 100644 --- a/dendrite/browser/async_api/_core/mixin/keyboard.py +++ b/dendrite/browser/async_api/_core/mixin/keyboard.py @@ -1,6 +1,7 @@ -from typing import Any, Union, Literal -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from typing import Literal, Union + from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol class KeyboardMixin(DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/mixin/markdown.py b/dendrite/browser/async_api/_core/mixin/markdown.py index 68e8adc..8bd1697 100644 --- a/dendrite/browser/async_api/_core/mixin/markdown.py +++ b/dendrite/browser/async_api/_core/mixin/markdown.py @@ -1,12 +1,12 @@ +import re from typing import Optional + from bs4 import BeautifulSoup -import re +from markdownify import markdownify as md from dendrite.browser.async_api._core.mixin.extract import ExtractionMixin from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol -from markdownify import markdownify as md - class MarkdownMixin(ExtractionMixin, DendritePageProtocol): async def markdown(self, prompt: Optional[str] = None): diff --git a/dendrite/browser/async_api/_core/mixin/wait_for.py b/dendrite/browser/async_api/_core/mixin/wait_for.py index 1a2b029..58ffacf 100644 --- a/dendrite/browser/async_api/_core/mixin/wait_for.py +++ b/dendrite/browser/async_api/_core/mixin/wait_for.py @@ -1,13 +1,14 @@ import asyncio import time +from loguru import logger +from dendrite.browser._common._exceptions.dendrite_exception import ( + DendriteException, + PageConditionNotMet, +) from dendrite.browser.async_api._core.mixin.ask import AskMixin from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import PageConditionNotMet -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException - -from loguru import logger class WaitForMixin(AskMixin, DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/models/api_config.py b/dendrite/browser/async_api/_core/models/api_config.py deleted file mode 100644 index 43f652a..0000000 --- a/dendrite/browser/async_api/_core/models/api_config.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import Optional -from pydantic import BaseModel, model_validator - -from dendrite.browser._common._exceptions.dendrite_exception import MissingApiKeyError - - -class APIConfig(BaseModel): - """ - Configuration model for API keys used in the Dendrite SDK. - - Attributes: - dendrite_api_key (Optional[str]): The API key for Dendrite services. - openai_api_key (Optional[str]): The API key for OpenAI services. If you wish to use your own API key, you can do so by passing it to the AsyncDendrite. - anthropic_api_key (Optional[str]): The API key for Anthropic services. If you wish to use your own API key, you can do so by passing it to the AsyncDendrite. - - Raises: - ValueError: If a valid dendrite_api_key is not provided. - """ - - dendrite_api_key: Optional[str] = None - openai_api_key: Optional[str] = None - anthropic_api_key: Optional[str] = None - - @model_validator(mode="before") - def _check_api_keys(cls, values): - dendrite_api_key = values.get("dendrite_api_key") - - if not dendrite_api_key: - raise MissingApiKeyError( - "A valid dendrite_api_key must be provided. Make sure you have set the DENDRITE_API_KEY environment variable or passed it to the AsyncDendrite." - ) - - return values diff --git a/dendrite/browser/async_api/_core/models/authentication.py b/dendrite/browser/async_api/_core/models/authentication.py index 3c2656e..56992c9 100644 --- a/dendrite/browser/async_api/_core/models/authentication.py +++ b/dendrite/browser/async_api/_core/models/authentication.py @@ -1,5 +1,6 @@ -from pydantic import BaseModel from typing import List, Literal, Optional + +from pydantic import BaseModel from typing_extensions import TypedDict diff --git a/dendrite/browser/async_api/_core/models/download_interface.py b/dendrite/browser/async_api/_core/models/download_interface.py index c38a486..bdb7ba9 100644 --- a/dendrite/browser/async_api/_core/models/download_interface.py +++ b/dendrite/browser/async_api/_core/models/download_interface.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from pathlib import Path from typing import Any, Union + from playwright.async_api import Download diff --git a/dendrite/browser/async_api/_core/models/page_diff_information.py b/dendrite/browser/async_api/_core/models/page_diff_information.py index dd37e9f..e69de29 100644 --- a/dendrite/browser/async_api/_core/models/page_diff_information.py +++ b/dendrite/browser/async_api/_core/models/page_diff_information.py @@ -1,7 +0,0 @@ -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class PageDiffInformation(BaseModel): - page_before: PageInformation - page_after: PageInformation diff --git a/dendrite/browser/async_api/_core/models/page_information.py b/dendrite/browser/async_api/_core/models/page_information.py index 67e1909..67d7892 100644 --- a/dendrite/browser/async_api/_core/models/page_information.py +++ b/dendrite/browser/async_api/_core/models/page_information.py @@ -1,6 +1,7 @@ -from typing import Dict, Optional -from typing_extensions import TypedDict +from typing import Optional + from pydantic import BaseModel +from typing_extensions import TypedDict class InteractableElementInfo(TypedDict): diff --git a/dendrite/browser/async_api/_core/protocol/page_protocol.py b/dendrite/browser/async_api/_core/protocol/page_protocol.py index c927a08..915f0fc 100644 --- a/dendrite/browser/async_api/_core/protocol/page_protocol.py +++ b/dendrite/browser/async_api/_core/protocol/page_protocol.py @@ -1,10 +1,11 @@ from typing import TYPE_CHECKING, Protocol -from dendrite.browser.async_api._api.browser_api_client import BrowserAPIClient +from dendrite.logic.hosted._api.browser_api_client import BrowserAPIClient +from dendrite.logic.interfaces.async_api import LocalProtocol if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_page import AsyncPage from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from dendrite.browser.async_api._core.dendrite_page import AsyncPage class DendritePageProtocol(Protocol): @@ -15,6 +16,6 @@ class DendritePageProtocol(Protocol): def _get_dendrite_browser(self) -> "AsyncDendrite": ... - def _get_browser_api_client(self) -> BrowserAPIClient: ... + def _get_logic_api(self) -> LocalProtocol: ... async def _get_page(self) -> "AsyncPage": ... diff --git a/dendrite/browser/async_api/_remote_impl/browserbase/_client.py b/dendrite/browser/async_api/_remote_impl/browserbase/_client.py index 79b7dce..b29b607 100644 --- a/dendrite/browser/async_api/_remote_impl/browserbase/_client.py +++ b/dendrite/browser/async_api/_remote_impl/browserbase/_client.py @@ -1,7 +1,8 @@ import asyncio -from pathlib import Path import time +from pathlib import Path from typing import Optional, Union + import httpx from loguru import logger diff --git a/dendrite/browser/async_api/_remote_impl/browserbase/_download.py b/dendrite/browser/async_api/_remote_impl/browserbase/_download.py index 965e932..354e9b3 100644 --- a/dendrite/browser/async_api/_remote_impl/browserbase/_download.py +++ b/dendrite/browser/async_api/_remote_impl/browserbase/_download.py @@ -1,13 +1,16 @@ -from pathlib import Path import re import shutil -from typing import Union import zipfile +from pathlib import Path +from typing import Union + from loguru import logger from playwright.async_api import Download from dendrite.browser.async_api._core.models.download_interface import DownloadInterface -from dendrite.browser.async_api._remote_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.async_api._remote_impl.browserbase._client import ( + BrowserbaseClient, +) class AsyncBrowserbaseDownload(DownloadInterface): diff --git a/dendrite/browser/async_api/_remote_impl/browserbase/_impl.py b/dendrite/browser/async_api/_remote_impl/browserbase/_impl.py index 04dc3ec..0d1b6ee 100644 --- a/dendrite/browser/async_api/_remote_impl/browserbase/_impl.py +++ b/dendrite/browser/async_api/_remote_impl/browserbase/_impl.py @@ -1,15 +1,21 @@ from typing import TYPE_CHECKING, Optional -from dendrite.browser._common._exceptions.dendrite_exception import BrowserNotLaunchedError + +from dendrite.browser._common._exceptions.dendrite_exception import ( + BrowserNotLaunchedError, +) from dendrite.browser.async_api._core._impl_browser import ImplBrowser from dendrite.browser.async_api._core._type_spec import PlaywrightPage from dendrite.browser.remote.browserbase_config import BrowserbaseConfig if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.browser.async_api._remote_impl.browserbase._client import BrowserbaseClient -from playwright.async_api import Playwright + from loguru import logger +from playwright.async_api import Playwright +from dendrite.browser.async_api._remote_impl.browserbase._client import ( + BrowserbaseClient, +) from dendrite.browser.async_api._remote_impl.browserbase._download import ( AsyncBrowserbaseDownload, ) diff --git a/dendrite/browser/async_api/_remote_impl/browserless/_impl.py b/dendrite/browser/async_api/_remote_impl/browserless/_impl.py index 625fcd0..61c8a42 100644 --- a/dendrite/browser/async_api/_remote_impl/browserless/_impl.py +++ b/dendrite/browser/async_api/_remote_impl/browserless/_impl.py @@ -1,17 +1,24 @@ import json from typing import TYPE_CHECKING, Optional -from dendrite.browser._common._exceptions.dendrite_exception import BrowserNotLaunchedError + +from dendrite.browser._common._exceptions.dendrite_exception import ( + BrowserNotLaunchedError, +) from dendrite.browser.async_api._core._impl_browser import ImplBrowser from dendrite.browser.async_api._core._type_spec import PlaywrightPage from dendrite.browser.remote.browserless_config import BrowserlessConfig if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from dendrite.browser.async_api._remote_impl.browserbase._client import BrowserbaseClient -from playwright.async_api import Playwright -from loguru import logger + import urllib.parse +from loguru import logger +from playwright.async_api import Playwright + +from dendrite.browser.async_api._remote_impl.browserbase._client import ( + BrowserbaseClient, +) from dendrite.browser.async_api._remote_impl.browserbase._download import ( AsyncBrowserbaseDownload, ) diff --git a/dendrite/browser/remote/__init__.py b/dendrite/browser/remote/__init__.py index d5c785b..b37ef34 100644 --- a/dendrite/browser/remote/__init__.py +++ b/dendrite/browser/remote/__init__.py @@ -1,7 +1,7 @@ from typing import Union -from dendrite.browser.remote.browserless_config import BrowserlessConfig -from dendrite.browser.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser.remote.browserless_config import BrowserlessConfig Providers = Union[BrowserbaseConfig, BrowserlessConfig] diff --git a/dendrite/browser/remote/browserbase_config.py b/dendrite/browser/remote/browserbase_config.py index b526b52..f86b02c 100644 --- a/dendrite/browser/remote/browserbase_config.py +++ b/dendrite/browser/remote/browserbase_config.py @@ -1,5 +1,6 @@ import os from typing import Optional + from dendrite.exceptions import MissingApiKeyError diff --git a/dendrite/browser/remote/provider.py b/dendrite/browser/remote/provider.py index 87890a9..fd615b0 100644 --- a/dendrite/browser/remote/provider.py +++ b/dendrite/browser/remote/provider.py @@ -1,11 +1,9 @@ from pathlib import Path from typing import Union - from dendrite.browser.remote import Providers from dendrite.browser.remote.browserbase_config import BrowserbaseConfig - try: import tomllib # type: ignore except ModuleNotFoundError: diff --git a/dendrite/browser/sync_api/__init__.py b/dendrite/browser/sync_api/__init__.py deleted file mode 100644 index 3085e23..0000000 --- a/dendrite/browser/sync_api/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from loguru import logger -from ._core.dendrite_browser import Dendrite -from ._core.dendrite_element import Element -from ._core.dendrite_page import Page -from ._core.models.response import ElementsResponse - -__all__ = ["Dendrite", "Element", "Page", "ElementsResponse"] diff --git a/dendrite/browser/sync_api/_api/__init__.py b/dendrite/browser/sync_api/_api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_api/_http_client.py b/dendrite/browser/sync_api/_api/_http_client.py deleted file mode 100644 index 2d053d8..0000000 --- a/dendrite/browser/sync_api/_api/_http_client.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -from typing import Optional -import httpx -from loguru import logger -from dendrite.browser.sync_api._core.models.api_config import APIConfig - - -class HTTPClient: - - def __init__(self, api_config: APIConfig, session_id: Optional[str] = None): - self.api_key = api_config.dendrite_api_key - self.session_id = session_id - self.base_url = self.resolve_base_url() - - def resolve_base_url(self): - base_url = ( - "http://localhost:8000/api/v1" - if os.environ.get("DENDRITE_DEV") - else "https://dendrite-server.azurewebsites.net/api/v1" - ) - return base_url - - def send_request( - self, - endpoint: str, - params: Optional[dict] = None, - data: Optional[dict] = None, - headers: Optional[dict] = None, - method: str = "GET", - ) -> httpx.Response: - url = f"{self.base_url}/{endpoint}" - headers = headers or {} - headers["Content-Type"] = "application/json" - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - if self.session_id: - headers["X-Session-ID"] = self.session_id - with httpx.Client(timeout=300) as client: - try: - response = client.request( - method, url, params=params, json=data, headers=headers - ) - response.raise_for_status() - return response - except httpx.HTTPStatusError as http_err: - logger.debug( - f"HTTP error occurred: {http_err.response.status_code}: {http_err.response.text}" - ) - raise - except httpx.ConnectError as connect_err: - logger.error( - f"Connection error occurred: {connect_err}. {url} Server might be down" - ) - raise - except httpx.RequestError as req_err: - raise - except Exception as err: - raise diff --git a/dendrite/browser/sync_api/_api/browser_api_client.py b/dendrite/browser/sync_api/_api/browser_api_client.py deleted file mode 100644 index c3ce4ab..0000000 --- a/dendrite/browser/sync_api/_api/browser_api_client.py +++ /dev/null @@ -1,100 +0,0 @@ -from typing import Optional -from loguru import logger -from dendrite.browser.sync_api._api.response.cache_extract_response import CacheExtractResponse -from dendrite.browser.sync_api._api.response.selector_cache_response import ( - SelectorCacheResponse, -) -from dendrite.browser.sync_api._core.models.authentication import AuthSession -from dendrite.browser.sync_api._api.response.get_element_response import GetElementResponse -from dendrite.browser.sync_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.browser.sync_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.browser.sync_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.browser.sync_api._api.dto.make_interaction_dto import MakeInteractionDTO -from dendrite.browser.sync_api._api.dto.extract_dto import ExtractDTO -from dendrite.browser.sync_api._api.dto.try_run_script_dto import TryRunScriptDTO -from dendrite.browser.sync_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.browser.sync_api._api.response.ask_page_response import AskPageResponse -from dendrite.browser.sync_api._api.response.interaction_response import InteractionResponse -from dendrite.browser.sync_api._api.response.extract_response import ExtractResponse -from dendrite.browser.sync_api._api._http_client import HTTPClient -from dendrite.browser._common._exceptions.dendrite_exception import InvalidAuthSessionError -from dendrite.browser.sync_api._api.dto.get_elements_dto import CheckSelectorCacheDTO - - -class BrowserAPIClient(HTTPClient): - - def authenticate(self, dto: AuthenticateDTO): - res = self.send_request( - "actions/authenticate", data=dto.model_dump(), method="POST" - ) - if res.status_code == 204: - raise InvalidAuthSessionError(domain=dto.domains) - return AuthSession(**res.json()) - - def upload_auth_session(self, dto: UploadAuthSessionDTO): - self.send_request("actions/upload-auth-session", data=dto.dict(), method="POST") - - def check_selector_cache(self, dto: CheckSelectorCacheDTO) -> SelectorCacheResponse: - res = self.send_request( - "actions/check-selector-cache", data=dto.dict(), method="POST" - ) - return SelectorCacheResponse(**res.json()) - - def get_interactions_selector(self, dto: GetElementsDTO) -> GetElementResponse: - res = self.send_request( - "actions/get-interaction-selector", data=dto.dict(), method="POST" - ) - return GetElementResponse(**res.json()) - - def make_interaction(self, dto: MakeInteractionDTO) -> InteractionResponse: - res = self.send_request( - "actions/make-interaction", data=dto.dict(), method="POST" - ) - res_dict = res.json() - return InteractionResponse( - status=res_dict["status"], message=res_dict["message"] - ) - - def check_extract_cache(self, dto: ExtractDTO) -> CacheExtractResponse: - res = self.send_request( - "actions/check-extract-cache", data=dto.dict(), method="POST" - ) - return CacheExtractResponse(**res.json()) - - def extract(self, dto: ExtractDTO) -> ExtractResponse: - res = self.send_request("actions/extract-page", data=dto.dict(), method="POST") - res_dict = res.json() - return ExtractResponse( - status=res_dict["status"], - message=res_dict["message"], - return_data=res_dict["return_data"], - created_script=res_dict.get("created_script", None), - used_cache=res_dict.get("used_cache", False), - ) - - def ask_page(self, dto: AskPageDTO) -> AskPageResponse: - res = self.send_request("actions/ask-page", data=dto.dict(), method="POST") - res_dict = res.json() - return AskPageResponse( - status=res_dict["status"], - description=res_dict["description"], - return_data=res_dict["return_data"], - ) - - def try_run_cached(self, dto: TryRunScriptDTO) -> Optional[ExtractResponse]: - res = self.send_request( - "actions/try-run-cached", data=dto.dict(), method="POST" - ) - if res is None: - return None - res_dict = res.json() - loaded_value = res_dict["return_data"] - if loaded_value is None: - return None - return ExtractResponse( - status=res_dict["status"], - message=res_dict["message"], - return_data=loaded_value, - created_script=res_dict.get("created_script", None), - used_cache=res_dict.get("used_cache", False), - ) diff --git a/dendrite/browser/sync_api/_api/dto/__init__.py b/dendrite/browser/sync_api/_api/dto/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_api/dto/ask_page_dto.py b/dendrite/browser/sync_api/_api/dto/ask_page_dto.py deleted file mode 100644 index 5b068e1..0000000 --- a/dendrite/browser/sync_api/_api/dto/ask_page_dto.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Any, Optional -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.api_config import APIConfig -from dendrite.browser.sync_api._core.models.page_information import PageInformation - - -class AskPageDTO(BaseModel): - prompt: str - return_schema: Optional[Any] - page_information: PageInformation - api_config: APIConfig diff --git a/dendrite/browser/sync_api/_api/dto/authenticate_dto.py b/dendrite/browser/sync_api/_api/dto/authenticate_dto.py deleted file mode 100644 index f5a1de7..0000000 --- a/dendrite/browser/sync_api/_api/dto/authenticate_dto.py +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Union -from pydantic import BaseModel - - -class AuthenticateDTO(BaseModel): - domains: Union[str, list[str]] diff --git a/dendrite/browser/sync_api/_api/dto/extract_dto.py b/dendrite/browser/sync_api/_api/dto/extract_dto.py deleted file mode 100644 index 01fccfc..0000000 --- a/dendrite/browser/sync_api/_api/dto/extract_dto.py +++ /dev/null @@ -1,24 +0,0 @@ -import json -from typing import Any -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.api_config import APIConfig -from dendrite.browser.sync_api._core.models.page_information import PageInformation - - -class ExtractDTO(BaseModel): - page_information: PageInformation - api_config: APIConfig - prompt: str - return_data_json_schema: Any - use_screenshot: bool = False - use_cache: bool = True - force_use_cache: bool = False - - @property - def combined_prompt(self) -> str: - json_schema_prompt = ( - "" - if self.return_data_json_schema is None - else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" - ) - return f"Task: {self.prompt}{json_schema_prompt}" diff --git a/dendrite/browser/sync_api/_api/dto/get_elements_dto.py b/dendrite/browser/sync_api/_api/dto/get_elements_dto.py deleted file mode 100644 index 8fe6a88..0000000 --- a/dendrite/browser/sync_api/_api/dto/get_elements_dto.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Dict, Union -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.api_config import APIConfig -from dendrite.browser.sync_api._core.models.page_information import PageInformation - - -class CheckSelectorCacheDTO(BaseModel): - url: str - prompt: Union[str, Dict[str, str]] - - -class GetElementsDTO(BaseModel): - page_information: PageInformation - prompt: Union[str, Dict[str, str]] - api_config: APIConfig - use_cache: bool = True - only_one: bool - force_use_cache: bool = False diff --git a/dendrite/browser/sync_api/_api/dto/get_interaction_dto.py b/dendrite/browser/sync_api/_api/dto/get_interaction_dto.py deleted file mode 100644 index bb04c19..0000000 --- a/dendrite/browser/sync_api/_api/dto/get_interaction_dto.py +++ /dev/null @@ -1,9 +0,0 @@ -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.api_config import APIConfig -from dendrite.browser.sync_api._core.models.page_information import PageInformation - - -class GetInteractionDTO(BaseModel): - page_information: PageInformation - api_config: APIConfig - prompt: str diff --git a/dendrite/browser/sync_api/_api/dto/get_session_dto.py b/dendrite/browser/sync_api/_api/dto/get_session_dto.py deleted file mode 100644 index 6414cc3..0000000 --- a/dendrite/browser/sync_api/_api/dto/get_session_dto.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class GetSessionDTO(BaseModel): - user_id: str - domain: str diff --git a/dendrite/browser/sync_api/_api/dto/google_search_dto.py b/dendrite/browser/sync_api/_api/dto/google_search_dto.py deleted file mode 100644 index b1d7fd4..0000000 --- a/dendrite/browser/sync_api/_api/dto/google_search_dto.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Optional -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.api_config import APIConfig -from dendrite.browser.sync_api._core.models.page_information import PageInformation - - -class GoogleSearchDTO(BaseModel): - query: str - country: Optional[str] = None - filter_results_prompt: Optional[str] = None - page_information: PageInformation - api_config: APIConfig diff --git a/dendrite/browser/sync_api/_api/dto/make_interaction_dto.py b/dendrite/browser/sync_api/_api/dto/make_interaction_dto.py deleted file mode 100644 index b2c3741..0000000 --- a/dendrite/browser/sync_api/_api/dto/make_interaction_dto.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import Literal, Optional -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.api_config import APIConfig -from dendrite.browser.sync_api._core.models.page_diff_information import PageDiffInformation - -InteractionType = Literal["click", "fill", "hover"] - - -class MakeInteractionDTO(BaseModel): - url: str - dendrite_id: str - interaction_type: InteractionType - value: Optional[str] = None - expected_outcome: Optional[str] - page_delta_information: PageDiffInformation - api_config: APIConfig diff --git a/dendrite/browser/sync_api/_api/dto/try_run_script_dto.py b/dendrite/browser/sync_api/_api/dto/try_run_script_dto.py deleted file mode 100644 index a2b99ea..0000000 --- a/dendrite/browser/sync_api/_api/dto/try_run_script_dto.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Any, Optional -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.api_config import APIConfig - - -class TryRunScriptDTO(BaseModel): - url: str - raw_html: str - api_config: APIConfig - prompt: str - db_prompt: Optional[str] = None - return_data_json_schema: Any diff --git a/dendrite/browser/sync_api/_api/dto/upload_auth_session_dto.py b/dendrite/browser/sync_api/_api/dto/upload_auth_session_dto.py deleted file mode 100644 index 72e5535..0000000 --- a/dendrite/browser/sync_api/_api/dto/upload_auth_session_dto.py +++ /dev/null @@ -1,7 +0,0 @@ -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.authentication import AuthSession, StorageState - - -class UploadAuthSessionDTO(BaseModel): - auth_data: AuthSession - storage_state: StorageState diff --git a/dendrite/browser/sync_api/_api/response/__init__.py b/dendrite/browser/sync_api/_api/response/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_api/response/ask_page_response.py b/dendrite/browser/sync_api/_api/response/ask_page_response.py deleted file mode 100644 index 8d99ddc..0000000 --- a/dendrite/browser/sync_api/_api/response/ask_page_response.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Generic, Literal, TypeVar -from pydantic import BaseModel - -T = TypeVar("T") - - -class AskPageResponse(BaseModel, Generic[T]): - status: Literal["success", "error"] - return_data: T - description: str diff --git a/dendrite/browser/sync_api/_api/response/cache_extract_response.py b/dendrite/browser/sync_api/_api/response/cache_extract_response.py deleted file mode 100644 index 463d03b..0000000 --- a/dendrite/browser/sync_api/_api/response/cache_extract_response.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class CacheExtractResponse(BaseModel): - exists: bool diff --git a/dendrite/browser/sync_api/_api/response/extract_response.py b/dendrite/browser/sync_api/_api/response/extract_response.py deleted file mode 100644 index 7cfbbb2..0000000 --- a/dendrite/browser/sync_api/_api/response/extract_response.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import Generic, Optional, TypeVar -from pydantic import BaseModel -from dendrite.browser.sync_api._common.status import Status - -T = TypeVar("T") - - -class ExtractResponse(BaseModel, Generic[T]): - return_data: T - message: str - created_script: Optional[str] = None - status: Status - used_cache: bool diff --git a/dendrite/browser/sync_api/_api/response/get_element_response.py b/dendrite/browser/sync_api/_api/response/get_element_response.py deleted file mode 100644 index 9401e09..0000000 --- a/dendrite/browser/sync_api/_api/response/get_element_response.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Dict, List, Optional, Union -from pydantic import BaseModel -from dendrite.browser.sync_api._common.status import Status - - -class GetElementResponse(BaseModel): - status: Status - selectors: Optional[Union[List[str], Dict[str, List[str]]]] = None - message: str = "" - used_cache: bool = False diff --git a/dendrite/browser/sync_api/_api/response/google_search_response.py b/dendrite/browser/sync_api/_api/response/google_search_response.py deleted file mode 100644 index d435b71..0000000 --- a/dendrite/browser/sync_api/_api/response/google_search_response.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class SearchResult(BaseModel): - url: str - title: str - description: str - - -class GoogleSearchResponse(BaseModel): - results: List[SearchResult] diff --git a/dendrite/browser/sync_api/_api/response/selector_cache_response.py b/dendrite/browser/sync_api/_api/response/selector_cache_response.py deleted file mode 100644 index 4c0e388..0000000 --- a/dendrite/browser/sync_api/_api/response/selector_cache_response.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class SelectorCacheResponse(BaseModel): - exists: bool diff --git a/dendrite/browser/sync_api/_api/response/session_response.py b/dendrite/browser/sync_api/_api/response/session_response.py deleted file mode 100644 index 2d03b97..0000000 --- a/dendrite/browser/sync_api/_api/response/session_response.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class SessionResponse(BaseModel): - cookies: List[dict] - origins_storage: List[dict] diff --git a/dendrite/browser/sync_api/_common/__init__.py b/dendrite/browser/sync_api/_common/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_common/constants.py b/dendrite/browser/sync_api/_common/constants.py deleted file mode 100644 index ee49898..0000000 --- a/dendrite/browser/sync_api/_common/constants.py +++ /dev/null @@ -1,66 +0,0 @@ -STEALTH_ARGS = [ - "--no-pings", - "--mute-audio", - "--no-first-run", - "--no-default-browser-check", - "--disable-cloud-import", - "--disable-gesture-typing", - "--disable-offer-store-unmasked-wallet-cards", - "--disable-offer-upload-credit-cards", - "--disable-print-preview", - "--disable-voice-input", - "--disable-wake-on-wifi", - "--disable-cookie-encryption", - "--ignore-gpu-blocklist", - "--enable-async-dns", - "--enable-simple-cache-backend", - "--enable-tcp-fast-open", - "--prerender-from-omnibox=disabled", - "--enable-web-bluetooth", - "--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees", - "--aggressive-cache-discard", - "--disable-extensions", - "--disable-ipc-flooding-protection", - "--disable-blink-features=AutomationControlled", - "--test-type", - "--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance", - "--disable-component-extensions-with-background-pages", - "--disable-default-apps", - "--disable-breakpad", - "--disable-component-update", - "--disable-domain-reliability", - "--disable-sync", - "--disable-client-side-phishing-detection", - "--disable-hang-monitor", - "--disable-popup-blocking", - "--disable-prompt-on-repost", - "--metrics-recording-only", - "--safebrowsing-disable-auto-update", - "--password-store=basic", - "--autoplay-policy=no-user-gesture-required", - "--use-mock-keychain", - "--force-webrtc-ip-handling-policy=disable_non_proxied_udp", - "--webrtc-ip-handling-policy=disable_non_proxied_udp", - "--disable-session-crashed-bubble", - "--disable-crash-reporter", - "--disable-dev-shm-usage", - "--force-color-profile=srgb", - "--disable-translate", - "--disable-background-networking", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-infobars", - "--hide-scrollbars", - "--disable-renderer-backgrounding", - "--font-render-hinting=none", - "--disable-logging", - "--enable-surface-synchronization", - "--disable-threaded-animation", - "--disable-threaded-scrolling", - "--disable-checker-imaging", - "--disable-new-content-rendering-timeout", - "--disable-image-animation-resync", - "--disable-partial-raster", - "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4", - "--disable-layer-tree-host-memory-pressure", -] diff --git a/dendrite/browser/sync_api/_common/event_sync.py b/dendrite/browser/sync_api/_common/event_sync.py deleted file mode 100644 index 162bb8e..0000000 --- a/dendrite/browser/sync_api/_common/event_sync.py +++ /dev/null @@ -1,45 +0,0 @@ -import time -import time -from typing import Generic, Optional, Type, TypeVar, Union, cast -from playwright.sync_api import Page, Download, FileChooser - -Events = TypeVar("Events", Download, FileChooser) -mapping = {Download: "download", FileChooser: "filechooser"} - - -class EventSync(Generic[Events]): - - def __init__(self, event_type: Type[Events]): - self.event_type = event_type - self.event_set = False - self.data: Optional[Events] = None - - def get_data(self, pw_page: Page, timeout: float = 30000) -> Events: - start_time = time.time() - while not self.event_set: - elapsed_time = (time.time() - start_time) * 1000 - if elapsed_time > timeout: - raise TimeoutError(f'Timeout waiting for event "{self.event_type}".') - pw_page.wait_for_timeout(0) - time.sleep(0.01) - data = self.data - self.data = None - self.event_set = False - if data is None: - raise ValueError("Data is None for event type: ", self.event_type) - return data - - def set_event(self, data: Events) -> None: - """ - Sets the event and stores the provided data. - - This method is used to signal that the data is ready to be retrieved by any waiting tasks. - - Args: - data (T): The data to be stored and associated with the event. - - Returns: - None - """ - self.data = data - self.event_set = True diff --git a/dendrite/browser/sync_api/_common/status.py b/dendrite/browser/sync_api/_common/status.py deleted file mode 100644 index 427449d..0000000 --- a/dendrite/browser/sync_api/_common/status.py +++ /dev/null @@ -1,3 +0,0 @@ -from typing import Literal - -Status = Literal["success", "failed", "loading", "impossible"] diff --git a/dendrite/browser/sync_api/_core/__init__.py b/dendrite/browser/sync_api/_core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_core/_impl_browser.py b/dendrite/browser/sync_api/_core/_impl_browser.py deleted file mode 100644 index 67899cd..0000000 --- a/dendrite/browser/sync_api/_core/_impl_browser.py +++ /dev/null @@ -1,85 +0,0 @@ -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from playwright.sync_api import Download, Browser, Playwright - - -class ImplBrowser(ABC): - - @abstractmethod - def __init__(self, settings): - pass - - @abstractmethod - def get_download( - self, dendrite_browser: "Dendrite", pw_page: PlaywrightPage, timeout: float - ) -> Download: - """ - Retrieves the download event from the browser. - - Returns: - Download: The download event. - - Raises: - Exception: If there is an issue retrieving the download event. - """ - pass - - @abstractmethod - def start_browser(self, playwright: Playwright, pw_options: dict) -> Browser: - """ - Starts the browser session. - - Returns: - Browser: The browser session. - - Raises: - Exception: If there is an issue starting the browser session. - """ - pass - - @abstractmethod - def configure_context(self, browser: "Dendrite") -> None: - """ - Configures the browser context. - - Args: - browser (Dendrite): The browser to configure. - - Raises: - Exception: If there is an issue configuring the browser context. - """ - pass - - @abstractmethod - def stop_session(self) -> None: - """ - Stops the browser session. - - Raises: - Exception: If there is an issue stopping the browser session. - """ - pass - - -class LocalImpl(ImplBrowser): - - def __init__(self) -> None: - pass - - def start_browser(self, playwright: Playwright, pw_options) -> Browser: - return playwright.chromium.launch(**pw_options) - - def get_download( - self, dendrite_browser: "Dendrite", pw_page: PlaywrightPage, timeout: float - ) -> Download: - return dendrite_browser._download_handler.get_data(pw_page, timeout) - - def configure_context(self, browser: "Dendrite"): - pass - - def stop_session(self): - pass diff --git a/dendrite/browser/sync_api/_core/_impl_mapping.py b/dendrite/browser/sync_api/_core/_impl_mapping.py deleted file mode 100644 index 8bc4e62..0000000 --- a/dendrite/browser/sync_api/_core/_impl_mapping.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Any, Dict, Optional, Type -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser, LocalImpl -from dendrite.browser.sync_api._ext_impl.browserbase._impl import BrowserBaseImpl -from dendrite.browser.sync_api._ext_impl.browserless._impl import BrowserlessImpl -from dendrite.browser.remote.browserless_config import BrowserlessConfig -from dendrite.browser.remote.browserbase_config import BrowserbaseConfig -from dendrite.browser.remote import Providers - -IMPL_MAPPING: Dict[Type[Providers], Type[ImplBrowser]] = { - BrowserbaseConfig: BrowserBaseImpl, - BrowserlessConfig: BrowserlessImpl, -} -SETTINGS_CLASSES: Dict[str, Type[Providers]] = { - "browserbase": BrowserbaseConfig, - "browserless": BrowserlessConfig, -} - - -def get_impl(remote_provider: Optional[Providers]) -> ImplBrowser: - if remote_provider is None: - return LocalImpl() - try: - provider_class = IMPL_MAPPING[type(remote_provider)] - except KeyError: - raise ValueError( - f"No implementation for {type(remote_provider)}. Available providers: {', '.join(map(lambda x: x.__name__, IMPL_MAPPING.keys()))}" - ) - return provider_class(remote_provider) diff --git a/dendrite/browser/sync_api/_core/_js/__init__.py b/dendrite/browser/sync_api/_core/_js/__init__.py deleted file mode 100644 index ccaf080..0000000 --- a/dendrite/browser/sync_api/_core/_js/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from pathlib import Path - - -def load_script(filename: str) -> str: - current_dir = Path(__file__).parent - file_path = current_dir / filename - return file_path.read_text(encoding="utf-8") - - -GENERATE_DENDRITE_IDS_SCRIPT = load_script("generateDendriteIDs.js") -GENERATE_DENDRITE_IDS_IFRAME_SCRIPT = load_script("generateDendriteIDsIframe.js") diff --git a/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js b/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js deleted file mode 100644 index 7f03d55..0000000 --- a/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js +++ /dev/null @@ -1,90 +0,0 @@ -// Save the original methods before redefining them -EventTarget.prototype._originalAddEventListener = EventTarget.prototype.addEventListener; -EventTarget.prototype._originalRemoveEventListener = EventTarget.prototype.removeEventListener; - -// Redefine the addEventListener method -EventTarget.prototype.addEventListener = function(event, listener, options = false) { - // Initialize the eventListenerList if it doesn't exist - if (!this.eventListenerList) { - this.eventListenerList = {}; - } - // Initialize the event list for the specific event if it doesn't exist - if (!this.eventListenerList[event]) { - this.eventListenerList[event] = []; - } - // Add the event listener details to the event list - this.eventListenerList[event].push({ listener, options, outerHTML: this.outerHTML }); - - // Call the original addEventListener method - this._originalAddEventListener(event, listener, options); -}; - -// Redefine the removeEventListener method -EventTarget.prototype.removeEventListener = function(event, listener, options = false) { - // Remove the event listener details from the event list - if (this.eventListenerList && this.eventListenerList[event]) { - this.eventListenerList[event] = this.eventListenerList[event].filter( - item => item.listener !== listener - ); - } - - // Call the original removeEventListener method - this._originalRemoveEventListener( event, listener, options); -}; - -// Get event listeners for a specific event type or all events if not specified -EventTarget.prototype._getEventListeners = function(eventType) { - if (!this.eventListenerList) { - this.eventListenerList = {}; - } - - const eventsToCheck = ['click', 'dblclick', 'mousedown', 'mouseup', 'mouseover', 'mouseout', 'mousemove', 'keydown', 'keyup', 'keypress']; - - eventsToCheck.forEach(type => { - if (!eventType || eventType === type) { - if (this[`on${type}`]) { - if (!this.eventListenerList[type]) { - this.eventListenerList[type] = []; - } - this.eventListenerList[type].push({ listener: this[`on${type}`], inline: true }); - } - } - }); - - return eventType === undefined ? this.eventListenerList : this.eventListenerList[eventType]; -}; - -// Utility to show events -function _showEvents(events) { - let result = ''; - for (let event in events) { - result += `${event} ----------------> ${events[event].length}\n`; - for (let listenerObj of events[event]) { - result += `${listenerObj.listener.toString()}\n`; - } - } - return result; -} - -// Extend EventTarget prototype with utility methods -EventTarget.prototype.on = function(event, callback, options) { - this.addEventListener(event, callback, options); - return this; -}; - -EventTarget.prototype.off = function(event, callback, options) { - this.removeEventListener(event, callback, options); - return this; -}; - -EventTarget.prototype.emit = function(event, args = null) { - this.dispatchEvent(new CustomEvent(event, { detail: args })); - return this; -}; - -// Make these methods non-enumerable -Object.defineProperties(EventTarget.prototype, { - on: { enumerable: false }, - off: { enumerable: false }, - emit: { enumerable: false } -}); diff --git a/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js b/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js deleted file mode 100644 index 3ad8574..0000000 --- a/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js +++ /dev/null @@ -1,85 +0,0 @@ -var hashCode = (string) => { - var hash = 0, i, chr; - if (string.length === 0) return hash; - for (i = 0; i < string.length; i++) { - chr = string.charCodeAt(i); - hash = ((hash << 5) - hash) + chr; - hash |= 0; // Convert to 32bit integer - } - return hash; -} - -var getXPathForElement = (element) => { - const getElementIndex = (element) => { - let index = 1; - let sibling = element.previousElementSibling; - - while (sibling) { - if (sibling.localName === element.localName) { - index++; - } - sibling = sibling.previousElementSibling; - } - - return index; - }; - - const segs = elm => { - if (!elm || elm.nodeType !== 1) return ['']; - if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; - const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; - let index = getElementIndex(elm); - - return [...segs(elm.parentNode), `${localName}[${index}]`]; - }; - return segs(element).join('/'); -} - -// Create a Map to store used hashes and their counters -const usedHashes = new Map(); - -document.querySelectorAll('*').forEach((element, index) => { - try { - - const xpath = getXPathForElement(element); - const hash = hashCode(xpath); - const baseId = hash.toString(36); - - const markHidden = (hidden_element) => { - // Mark the hidden element itself - hidden_element.setAttribute('data-hidden', 'true'); - - } - - // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; - const isHidden = !element.checkVisibility(); - // computedStyle.width === '0px' || - // computedStyle.height === '0px'; - - if (isHidden) { - markHidden(element); - }else{ - element.removeAttribute("data-hidden") // in case we hid it in a previous call - } - - let uniqueId = baseId; - let counter = 0; - - // Check if this hash has been used before - while (usedHashes.has(uniqueId)) { - // If it has, increment the counter and create a new uniqueId - counter++; - uniqueId = `${baseId}_${counter}`; - } - - // Add the uniqueId to the usedHashes Map - usedHashes.set(uniqueId, true); - element.setAttribute('d-id', uniqueId); - } catch (error) { - // Fallback: use a hash of the tag name and index - const fallbackId = hashCode(`${element.tagName}_${index}`).toString(36); - console.error('Error processing element, using fallback:',fallbackId, element, error); - - element.setAttribute('d-id', `fallback_${fallbackId}`); - } -}); \ No newline at end of file diff --git a/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js b/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js deleted file mode 100644 index bb2a65d..0000000 --- a/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js +++ /dev/null @@ -1,90 +0,0 @@ -({frame_path}) => { - var hashCode = (string) => { - var hash = 0, i, chr; - if (string.length === 0) return hash; - for (i = 0; i < string.length; i++) { - chr = string.charCodeAt(i); - hash = ((hash << 5) - hash) + chr; - hash |= 0; // Convert to 32bit integer - } - return hash; - } - - var getXPathForElement = (element) => { - const getElementIndex = (element) => { - let index = 1; - let sibling = element.previousElementSibling; - - while (sibling) { - if (sibling.localName === element.localName) { - index++; - } - sibling = sibling.previousElementSibling; - } - - return index; - }; - - const segs = elm => { - if (!elm || elm.nodeType !== 1) return ['']; - if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; - const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; - let index = getElementIndex(elm); - - return [...segs(elm.parentNode), `${localName}[${index}]`]; - }; - return segs(element).join('/'); - } - - - // Create a Map to store used hashes and their counters - const usedHashes = new Map(); - - document.querySelectorAll('*').forEach((element, index) => { - try { - const markHidden = (hidden_element) => { - // Mark the hidden element itself - hidden_element.setAttribute('data-hidden', 'true'); - } - - // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; - const isHidden = !element.checkVisibility(); - // computedStyle.width === '0px' || - // computedStyle.height === '0px'; - - if (isHidden) { - markHidden(element); - }else{ - element.removeAttribute("data-hidden") // in case we hid it in a previous call - } - const xpath = getXPathForElement(element); - if(frame_path){ - element.setAttribute("iframe-path",frame_path) - xpath = frame_path + xpath; - } - const hash = hashCode(xpath); - const baseId = hash.toString(36); - - let uniqueId = baseId; - let counter = 0; - - // Check if this hash has been used before - while (usedHashes.has(uniqueId)) { - // If it has, increment the counter and create a new uniqueId - counter++; - uniqueId = `${baseId}_${counter}`; - } - - // Add the uniqueId to the usedHashes Map - usedHashes.set(uniqueId, true); - element.setAttribute('d-id', uniqueId); - } catch (error) { - // Fallback: use a hash of the tag name and index - const fallbackId = hashCode(`${element.tagName}_${index}`).toString(36); - console.error('Error processing element, using fallback:',fallbackId, element, error); - - element.setAttribute('d-id', `fallback_${fallbackId}`); - } - }); -} - diff --git a/dendrite/browser/sync_api/_core/_managers/__init__.py b/dendrite/browser/sync_api/_core/_managers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py b/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py deleted file mode 100644 index 8dc632b..0000000 --- a/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py +++ /dev/null @@ -1,67 +0,0 @@ -import time -import time -from typing import TYPE_CHECKING, Dict, Optional - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_page import Page - - -class NavigationTracker: - - def __init__(self, page: "Page"): - self.playwright_page = page.playwright_page - self._nav_start_timestamp: Optional[float] = None - self.playwright_page.on("framenavigated", self._on_frame_navigated) - self.playwright_page.on("popup", self._on_popup) - self._last_events: Dict[str, Optional[float]] = { - "framenavigated": None, - "popup": None, - } - - def _on_frame_navigated(self, frame): - self._last_events["framenavigated"] = time.time() - if frame is self.playwright_page.main_frame: - self._last_main_frame_url = frame.url - self._last_frame_navigated_timestamp = time.time() - - def _on_popup(self, page): - self._last_events["popup"] = time.time() - - def start_nav_tracking(self): - """Call this just before performing an action that might trigger navigation""" - self._nav_start_timestamp = time.time() - for event in self._last_events: - self._last_events[event] = None - - def get_nav_events_since_start(self): - """ - Returns which events have fired since start_nav_tracking() was called - and how long after the start they occurred - """ - if self._nav_start_timestamp is None: - return "Navigation tracking not started. Call start_nav_tracking() first." - results = {} - for event, timestamp in self._last_events.items(): - if timestamp is not None: - delay = timestamp - self._nav_start_timestamp - results[event] = f"{delay:.3f}s" - else: - results[event] = "not fired" - return results - - def has_navigated_since_start(self): - """Returns True if any navigation event has occurred since start_nav_tracking()""" - if self._nav_start_timestamp is None: - return False - start_time = time.time() - max_wait = 1.0 - while time.time() - start_time < max_wait: - if any( - ( - timestamp is not None and timestamp > self._nav_start_timestamp - for timestamp in self._last_events.values() - ) - ): - return True - time.sleep(0.1) - return False diff --git a/dendrite/browser/sync_api/_core/_managers/page_manager.py b/dendrite/browser/sync_api/_core/_managers/page_manager.py deleted file mode 100644 index 79734db..0000000 --- a/dendrite/browser/sync_api/_core/_managers/page_manager.py +++ /dev/null @@ -1,74 +0,0 @@ -from typing import Optional, TYPE_CHECKING -from loguru import logger -from playwright.sync_api import BrowserContext, Download, FileChooser - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.sync_api._core.dendrite_page import Page - - -class PageManager: - - def __init__(self, dendrite_browser, browser_context: BrowserContext): - self.pages: list[Page] = [] - self.active_page: Optional[Page] = None - self.browser_context = browser_context - self.dendrite_browser: Dendrite = dendrite_browser - browser_context.on("page", self._page_on_open_handler) - - def new_page(self) -> Page: - new_page = self.browser_context.new_page() - if self.active_page and new_page == self.active_page.playwright_page: - return self.active_page - client = self.dendrite_browser._get_browser_api_client() - dendrite_page = Page(new_page, self.dendrite_browser, client) - self.pages.append(dendrite_page) - self.active_page = dendrite_page - return dendrite_page - - def get_active_page(self) -> Page: - if self.active_page is None: - return self.new_page() - return self.active_page - - def _page_on_close_handler(self, page: PlaywrightPage): - if self.browser_context and (not self.dendrite_browser.closed): - copy_pages = self.pages.copy() - is_active_page = False - for dendrite_page in copy_pages: - if dendrite_page.playwright_page == page: - self.pages.remove(dendrite_page) - if dendrite_page == self.active_page: - is_active_page = True - break - for i in reversed(range(len(self.pages))): - try: - self.active_page = self.pages[i] - self.pages[i].playwright_page.bring_to_front() - break - except Exception as e: - logger.warning(f"Error switching to the next page: {e}") - continue - - def _page_on_crash_handler(self, page: PlaywrightPage): - logger.error(f"Page crashed: {page.url}") - page.reload() - - def _page_on_download_handler(self, download: Download): - logger.debug(f"Download started: {download.url}") - self.dendrite_browser._download_handler.set_event(download) - - def _page_on_filechooser_handler(self, file_chooser: FileChooser): - logger.debug("File chooser opened") - self.dendrite_browser._upload_handler.set_event(file_chooser) - - def _page_on_open_handler(self, page: PlaywrightPage): - page.on("close", self._page_on_close_handler) - page.on("crash", self._page_on_crash_handler) - page.on("download", self._page_on_download_handler) - page.on("filechooser", self._page_on_filechooser_handler) - client = self.dendrite_browser._get_browser_api_client() - dendrite_page = Page(page, self.dendrite_browser, client) - self.pages.append(dendrite_page) - self.active_page = dendrite_page diff --git a/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py b/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py deleted file mode 100644 index 9a01f7c..0000000 --- a/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py +++ /dev/null @@ -1,50 +0,0 @@ -import base64 -import os -from uuid import uuid4 -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage - - -class ScreenshotManager: - - def __init__(self, page: PlaywrightPage) -> None: - self.screenshot_before: str = "" - self.screenshot_after: str = "" - self.page = page - - def take_full_page_screenshot(self) -> str: - try: - scroll_height = self.page.evaluate( - "\n () => {\n const body = document.body;\n if (!body) {\n return 0; // Return 0 if body is null\n }\n return body.scrollHeight || 0;\n }\n " - ) - if scroll_height > 30000: - print( - f"Page height ({scroll_height}px) exceeds 30000px. Taking viewport screenshot instead." - ) - return self.take_viewport_screenshot() - image_data = self.page.screenshot( - type="jpeg", full_page=True, timeout=10000 - ) - except Exception as e: - print( - f"Full-page screenshot failed: {e}. Falling back to viewport screenshot." - ) - return self.take_viewport_screenshot() - if image_data is None: - return "" - return base64.b64encode(image_data).decode("utf-8") - - def take_viewport_screenshot(self) -> str: - image_data = self.page.screenshot(type="jpeg", timeout=10000) - if image_data is None: - return "" - reduced_base64 = base64.b64encode(image_data).decode("utf-8") - return reduced_base64 - - def store_screenshot(self, name, image_data): - if not name: - name = str(uuid4()) - filepath = os.path.join("test", f"{name}.jpeg") - os.makedirs(os.path.dirname(filepath), exist_ok=True) - with open(filepath, "wb") as file: - file.write(image_data) - return filepath diff --git a/dendrite/browser/sync_api/_core/_type_spec.py b/dendrite/browser/sync_api/_core/_type_spec.py deleted file mode 100644 index 5dfbb9b..0000000 --- a/dendrite/browser/sync_api/_core/_type_spec.py +++ /dev/null @@ -1,35 +0,0 @@ -import inspect -from typing import Any, Dict, Literal, Type, TypeVar, Union -from pydantic import BaseModel -from playwright.sync_api import Page - -Interaction = Literal["click", "fill", "hover"] -T = TypeVar("T") -PydanticModel = TypeVar("PydanticModel", bound=BaseModel) -PrimitiveTypes = PrimitiveTypes = Union[Type[bool], Type[int], Type[float], Type[str]] -JsonSchema = Dict[str, Any] -TypeSpec = Union[PrimitiveTypes, PydanticModel, JsonSchema] -PlaywrightPage = Page - - -def to_json_schema(type_spec: TypeSpec) -> Dict[str, Any]: - if isinstance(type_spec, dict): - return type_spec - if inspect.isclass(type_spec) and issubclass(type_spec, BaseModel): - return type_spec.model_json_schema() - if type_spec in (bool, int, float, str): - type_map = {bool: "boolean", int: "integer", float: "number", str: "string"} - return {"type": type_map[type_spec]} - raise ValueError(f"Unsupported type specification: {type_spec}") - - -def convert_to_type_spec(type_spec: TypeSpec, return_data: Any) -> TypeSpec: - if isinstance(type_spec, type): - if issubclass(type_spec, BaseModel): - return type_spec.model_validate(return_data) - if type_spec in (str, float, bool, int): - return type_spec(return_data) - raise ValueError(f"Unsupported type: {type_spec}") - if isinstance(type_spec, dict): - return return_data - raise ValueError(f"Unsupported type specification: {type_spec}") diff --git a/dendrite/browser/sync_api/_core/_utils.py b/dendrite/browser/sync_api/_core/_utils.py deleted file mode 100644 index 7993862..0000000 --- a/dendrite/browser/sync_api/_core/_utils.py +++ /dev/null @@ -1,101 +0,0 @@ -from typing import Optional, Union, List, TYPE_CHECKING -from playwright.sync_api import FrameLocator, ElementHandle, Error, Frame -from bs4 import BeautifulSoup -from loguru import logger -from dendrite.browser.sync_api._api.response.get_element_response import GetElementResponse -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.sync_api._core.dendrite_element import Element -from dendrite.browser.sync_api._core.models.response import ElementsResponse - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_page import Page -from dendrite.browser.sync_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT -from dendrite.browser.sync_api._dom.util.mild_strip import mild_strip_in_place - - -def expand_iframes(page: PlaywrightPage, page_soup: BeautifulSoup): - - def get_iframe_path(frame: Frame): - path_parts = [] - current_frame = frame - while current_frame.parent_frame is not None: - iframe_element = current_frame.frame_element() - iframe_id = iframe_element.get_attribute("d-id") - if iframe_id is None: - return None - path_parts.insert(0, iframe_id) - current_frame = current_frame.parent_frame - return "|".join(path_parts) - - for frame in page.frames: - if frame.parent_frame is None: - continue - iframe_element = frame.frame_element() - iframe_id = iframe_element.get_attribute("d-id") - if iframe_id is None: - continue - iframe_path = get_iframe_path(frame) - if iframe_path is None: - continue - try: - frame.evaluate( - GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, {"frame_path": iframe_path} - ) - frame_content = frame.content() - frame_tree = BeautifulSoup(frame_content, "lxml") - mild_strip_in_place(frame_tree) - merge_iframe_to_page(iframe_id, page_soup, frame_tree) - except Error as e: - logger.debug(f"Error processing frame {iframe_id}: {e}") - continue - - -def merge_iframe_to_page(iframe_id: str, page: BeautifulSoup, iframe: BeautifulSoup): - iframe_element = page.find("iframe", {"d-id": iframe_id}) - if iframe_element is None: - logger.debug(f"Could not find iframe with ID {iframe_id} in page soup") - return - iframe_element.replace_with(iframe) - - -def _get_all_elements_from_selector_soup( - selector: str, soup: BeautifulSoup, page: "Page" -) -> List[Element]: - dendrite_elements: List[Element] = [] - elements = soup.select(selector) - for element in elements: - frame = page._get_context(element) - d_id = element.get("d-id", "") - locator = frame.locator(f"xpath=//*[@d-id='{d_id}']") - if not d_id: - continue - if isinstance(d_id, list): - d_id = d_id[0] - dendrite_elements.append( - Element(d_id, locator, page.dendrite_browser, page._browser_api_client) - ) - return dendrite_elements - - -def get_elements_from_selectors_soup( - page: "Page", soup: BeautifulSoup, res: GetElementResponse, only_one: bool -) -> Union[Optional[Element], List[Element], ElementsResponse]: - if isinstance(res.selectors, dict): - result = {} - for key, selectors in res.selectors.items(): - for selector in selectors: - dendrite_elements = _get_all_elements_from_selector_soup( - selector, soup, page - ) - if len(dendrite_elements) > 0: - result[key] = dendrite_elements[0] - break - return ElementsResponse(result) - elif isinstance(res.selectors, list): - for selector in reversed(res.selectors): - dendrite_elements = _get_all_elements_from_selector_soup( - selector, soup, page - ) - if len(dendrite_elements) > 0: - return dendrite_elements[0] if only_one else dendrite_elements - return None diff --git a/dendrite/browser/sync_api/_core/dendrite_browser.py b/dendrite/browser/sync_api/_core/dendrite_browser.py deleted file mode 100644 index 06017c5..0000000 --- a/dendrite/browser/sync_api/_core/dendrite_browser.py +++ /dev/null @@ -1,428 +0,0 @@ -from abc import ABC, abstractmethod -import pathlib -import re -from typing import Any, List, Literal, Optional, Sequence, Union -from uuid import uuid4 -import os -from loguru import logger -from playwright.sync_api import ( - sync_playwright, - Playwright, - BrowserContext, - FileChooser, - Download, - Error, - FilePayload, -) -from dendrite.browser.sync_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.browser.sync_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.browser.sync_api._common.event_sync import EventSync -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser -from dendrite.browser.sync_api._core._impl_mapping import get_impl -from dendrite.browser.sync_api._core._managers.page_manager import PageManager -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.sync_api._core.dendrite_page import Page -from dendrite.browser.sync_api._common.constants import STEALTH_ARGS -from dendrite.browser.sync_api._core.mixin.ask import AskMixin -from dendrite.browser.sync_api._core.mixin.click import ClickMixin -from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin -from dendrite.browser.sync_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.sync_api._core.mixin.keyboard import KeyboardMixin -from dendrite.browser.sync_api._core.mixin.screenshot import ScreenshotMixin -from dendrite.browser.sync_api._core.mixin.wait_for import WaitForMixin -from dendrite.browser.sync_api._core.mixin.markdown import MarkdownMixin -from dendrite.browser.sync_api._core.models.authentication import AuthSession -from dendrite.browser.sync_api._core.models.api_config import APIConfig -from dendrite.browser.sync_api._api.browser_api_client import BrowserAPIClient -from dendrite.browser._common._exceptions.dendrite_exception import ( - BrowserNotLaunchedError, - DendriteException, - IncorrectOutcomeError, -) -from dendrite.browser.remote import Providers - - -class Dendrite( - ScreenshotMixin, - WaitForMixin, - MarkdownMixin, - ExtractionMixin, - AskMixin, - FillFieldsMixin, - ClickMixin, - KeyboardMixin, - GetElementMixin, - ABC, -): - """ - Dendrite is a class that manages a browser instance using Playwright, allowing - interactions with web pages using natural language. - - This class handles initialization with API keys for Dendrite, OpenAI, and Anthropic, manages browser - contexts, and provides methods for navigation, authentication, and other browser-related tasks. - - Attributes: - id (UUID): The unique identifier for the Dendrite instance. - auth_data (Optional[AuthSession]): The authentication session data for the browser. - dendrite_api_key (str): The API key for Dendrite, used for interactions with the Dendrite API. - playwright_options (dict): Options for configuring the Playwright browser instance. - playwright (Optional[Playwright]): The Playwright instance managing the browser. - browser_context (Optional[BrowserContext]): The current browser context, which may include cookies and other session data. - active_page_manager (Optional[PageManager]): The manager responsible for handling active pages within the browser context. - user_id (Optional[str]): The user ID associated with the browser session. - browser_api_client (BrowserAPIClient): The API client used for communicating with the Dendrite API. - api_config (APIConfig): The configuration for the language models, including API keys for OpenAI and Anthropic. - - Raises: - Exception: If any of the required API keys (Dendrite, OpenAI, Anthropic) are not provided or found in the environment variables. - """ - - def __init__( - self, - auth: Optional[Union[str, List[str]]] = None, - dendrite_api_key: Optional[str] = None, - openai_api_key: Optional[str] = None, - anthropic_api_key: Optional[str] = None, - playwright_options: Any = {"headless": False, "args": STEALTH_ARGS}, - remote_config: Optional[Providers] = None, - ): - """ - Initializes Dendrite with API keys and Playwright options. - - Args: - auth (Optional[Union[str, List[str]]]): The domains on which the browser should try and authenticate. - dendrite_api_key (Optional[str]): The Dendrite API key. If not provided, it's fetched from the environment variables. - openai_api_key (Optional[str]): Your own OpenAI API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. - anthropic_api_key (Optional[str]): The own Anthropic API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. - playwright_options (Any): Options for configuring Playwright. Defaults to running in non-headless mode with stealth arguments. - - Raises: - MissingApiKeyError: If the Dendrite API key is not provided or found in the environment variables. - """ - api_config = APIConfig( - dendrite_api_key=dendrite_api_key or os.environ.get("DENDRITE_API_KEY"), - openai_api_key=openai_api_key, - anthropic_api_key=anthropic_api_key, - ) - self._impl = self._get_impl(remote_config) - self.api_config = api_config - self.playwright: Optional[Playwright] = None - self.browser_context: Optional[BrowserContext] = None - self._id = uuid4().hex - self._playwright_options = playwright_options - self._active_page_manager: Optional[PageManager] = None - self._user_id: Optional[str] = None - self._upload_handler = EventSync(event_type=FileChooser) - self._download_handler = EventSync(event_type=Download) - self.closed = False - self._auth = auth - self._browser_api_client = BrowserAPIClient(api_config, self._id) - - @property - def pages(self) -> List[Page]: - """ - Retrieves the list of active pages managed by the PageManager. - - Returns: - List[Page]: The list of active pages. - """ - if self._active_page_manager: - return self._active_page_manager.pages - else: - raise BrowserNotLaunchedError() - - def _get_page(self) -> Page: - active_page = self.get_active_page() - return active_page - - def _get_browser_api_client(self) -> BrowserAPIClient: - return self._browser_api_client - - def _get_dendrite_browser(self) -> "Dendrite": - return self - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def _get_impl(self, remote_provider: Optional[Providers]) -> ImplBrowser: - return get_impl(remote_provider) - - def _get_auth_session(self, domains: Union[str, list[str]]): - dto = AuthenticateDTO(domains=domains) - auth_session: AuthSession = self._browser_api_client.authenticate(dto) - return auth_session - - def get_active_page(self) -> Page: - """ - Retrieves the currently active page managed by the PageManager. - - Returns: - Page: The active page object. - - Raises: - Exception: If there is an issue retrieving the active page. - """ - active_page_manager = self._get_active_page_manager() - return active_page_manager.get_active_page() - - def new_tab( - self, url: str, timeout: Optional[float] = 15000, expected_page: str = "" - ) -> Page: - """ - Opens a new tab and navigates to the specified URL. - - Args: - url (str): The URL to navigate to. - timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. - expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. - - Returns: - Page: The page object after navigation. - - Raises: - Exception: If there is an error during navigation or if the expected page type is not found. - """ - return self.goto( - url, new_tab=True, timeout=timeout, expected_page=expected_page - ) - - def goto( - self, - url: str, - new_tab: bool = False, - timeout: Optional[float] = 15000, - expected_page: str = "", - ) -> Page: - """ - Navigates to the specified URL, optionally in a new tab - - Args: - url (str): The URL to navigate to. - new_tab (bool, optional): Whether to open the URL in a new tab. Defaults to False. - timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. - expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. - - Returns: - Page: The page object after navigation. - - Raises: - Exception: If there is an error during navigation or if the expected page type is not found. - """ - if not re.match("^\\w+://", url): - url = f"https://{url}" - active_page_manager = self._get_active_page_manager() - if new_tab: - active_page = active_page_manager.new_page() - else: - active_page = active_page_manager.get_active_page() - try: - logger.info(f"Going to {url}") - active_page.playwright_page.goto(url, timeout=timeout) - except TimeoutError: - logger.debug("Timeout when loading page but continuing anyways.") - except Exception as e: - logger.debug(f"Exception when loading page but continuing anyways. {e}") - if expected_page != "": - try: - prompt = f"We are checking if we have arrived on the expected type of page. If it is apparent that we have arrived on the wrong page, output an error. Here is the description: '{expected_page}'" - active_page.ask(prompt, bool) - except DendriteException as e: - raise IncorrectOutcomeError(f"Incorrect navigation, reason: {e}") - return active_page - - def scroll_to_bottom( - self, - timeout: float = 30000, - scroll_increment: int = 1000, - no_progress_limit: int = 3, - ): - """ - Scrolls to the bottom of the current page. - - Returns: - None - """ - active_page = self.get_active_page() - active_page.scroll_to_bottom( - timeout=timeout, - scroll_increment=scroll_increment, - no_progress_limit=no_progress_limit, - ) - - def _launch(self): - """ - Launches the Playwright instance and sets up the browser context and page manager. - - This method initializes the Playwright instance, creates a browser context, and sets up the PageManager. - It also applies any authentication data if available. - - Returns: - Tuple[Browser, BrowserContext, PageManager]: The launched browser, context, and page manager. - - Raises: - Exception: If there is an issue launching the browser or setting up the context. - """ - os.environ["PW_TEST_SCREENSHOT_NO_FONTS_READY"] = "1" - self._playwright = sync_playwright().start() - browser = self._impl.start_browser(self._playwright, self._playwright_options) - if self._auth: - auth_session = self._get_auth_session(self._auth) - self.browser_context = browser.new_context( - storage_state=auth_session.to_storage_state(), - user_agent=auth_session.user_agent, - ) - else: - self.browser_context = ( - browser.contexts[0] - if len(browser.contexts) > 0 - else browser.new_context() - ) - self._active_page_manager = PageManager(self, self.browser_context) - self._impl.configure_context(self) - return (browser, self.browser_context, self._active_page_manager) - - def add_cookies(self, cookies): - """ - Adds cookies to the current browser context. - - Args: - cookies (List[Dict[str, Any]]): A list of cookies to be added to the browser context. - - Raises: - Exception: If the browser context is not initialized. - """ - if not self.browser_context: - raise DendriteException("Browser context not initialized") - self.browser_context.add_cookies(cookies) - - def close(self): - """ - Closes the browser and uploads authentication session data if available. - - This method stops the Playwright instance, closes the browser context, and uploads any - stored authentication session data if applicable. - - Returns: - None - - Raises: - Exception: If there is an issue closing the browser or uploading session data. - """ - self.closed = True - try: - if self.browser_context: - if self._auth: - auth_session = self._get_auth_session(self._auth) - storage_state = self.browser_context.storage_state() - dto = UploadAuthSessionDTO( - auth_data=auth_session, storage_state=storage_state - ) - self._browser_api_client.upload_auth_session(dto) - self._impl.stop_session() - self.browser_context.close() - except Error: - pass - try: - if self._playwright: - self._playwright.stop() - except AttributeError: - pass - except Exception: - pass - - def _is_launched(self): - """ - Checks whether the browser context has been launched. - - Returns: - bool: True if the browser context is launched, False otherwise. - """ - return self.browser_context is not None - - def _get_active_page_manager(self) -> PageManager: - """ - Retrieves the active PageManager instance, launching the browser if necessary. - - Returns: - PageManager: The active PageManager instance. - - Raises: - Exception: If there is an issue launching the browser or retrieving the PageManager. - """ - if not self._active_page_manager: - (_, _, active_page_manager) = self._launch() - return active_page_manager - return self._active_page_manager - - def get_download(self, timeout: float) -> Download: - """ - Retrieves the download event from the browser. - - Returns: - Download: The download event. - - Raises: - Exception: If there is an issue retrieving the download event. - """ - active_page = self.get_active_page() - pw_page = active_page.playwright_page - return self._get_download(pw_page, timeout) - - def _get_download(self, pw_page: PlaywrightPage, timeout: float) -> Download: - """ - Retrieves the download event from the browser. - - Returns: - Download: The download event. - - Raises: - Exception: If there is an issue retrieving the download event. - """ - return self._download_handler.get_data(pw_page, timeout=timeout) - - def upload_files( - self, - files: Union[ - str, - pathlib.Path, - FilePayload, - Sequence[Union[str, pathlib.Path]], - Sequence[FilePayload], - ], - timeout: float = 30000, - ) -> None: - """ - Uploads files to the active page using a file chooser. - - Args: - files (Union[str, pathlib.Path, FilePayload, Sequence[Union[str, pathlib.Path]], Sequence[FilePayload]]): The file(s) to be uploaded. - This can be a file path, a `FilePayload` object, or a sequence of file paths or `FilePayload` objects. - timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the file chooser to be ready. Defaults to 30. - - Returns: - None - """ - page = self.get_active_page() - file_chooser = self._get_filechooser(page.playwright_page, timeout) - file_chooser.set_files(files) - - def _get_filechooser( - self, pw_page: PlaywrightPage, timeout: float = 30000 - ) -> FileChooser: - """ - Uploads files to the browser. - - Args: - timeout (float): The maximum time to wait for the file chooser dialog. Defaults to 30000 milliseconds. - - Returns: - FileChooser: The file chooser dialog. - - Raises: - Exception: If there is an issue uploading files. - """ - return self._upload_handler.get_data(pw_page, timeout=timeout) diff --git a/dendrite/browser/sync_api/_core/dendrite_element.py b/dendrite/browser/sync_api/_core/dendrite_element.py deleted file mode 100644 index 7242768..0000000 --- a/dendrite/browser/sync_api/_core/dendrite_element.py +++ /dev/null @@ -1,237 +0,0 @@ -from __future__ import annotations -import time -import base64 -import functools -import time -from typing import TYPE_CHECKING, Optional -from loguru import logger -from playwright.sync_api import Locator -from dendrite.browser.sync_api._api.browser_api_client import BrowserAPIClient -from dendrite.browser._common._exceptions.dendrite_exception import IncorrectOutcomeError - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser.sync_api._core._managers.navigation_tracker import NavigationTracker -from dendrite.browser.sync_api._core.models.page_diff_information import PageDiffInformation -from dendrite.browser.sync_api._core._type_spec import Interaction -from dendrite.browser.sync_api._api.response.interaction_response import InteractionResponse -from dendrite.browser.sync_api._api.dto.make_interaction_dto import MakeInteractionDTO - - -def perform_action(interaction_type: Interaction): - """ - Decorator for performing actions on DendriteElements. - - This decorator wraps methods of Element to handle interactions, - expected outcomes, and error handling. - - Args: - interaction_type (Interaction): The type of interaction being performed. - - Returns: - function: The decorated function. - """ - - def decorator(func): - - @functools.wraps(func) - def wrapper(self: Element, *args, **kwargs) -> InteractionResponse: - expected_outcome: Optional[str] = kwargs.pop("expected_outcome", None) - if not expected_outcome: - func(self, *args, **kwargs) - return InteractionResponse(status="success", message="") - api_config = self._dendrite_browser.api_config - page_before = self._dendrite_browser.get_active_page() - page_before_info = page_before.get_page_information() - func(self, *args, expected_outcome=expected_outcome, **kwargs) - self._wait_for_page_changes(page_before.url) - page_after = self._dendrite_browser.get_active_page() - page_after_info = page_after.get_page_information() - page_delta_information = PageDiffInformation( - page_before=page_before_info, page_after=page_after_info - ) - dto = MakeInteractionDTO( - url=page_before.url, - dendrite_id=self.dendrite_id, - interaction_type=interaction_type, - expected_outcome=expected_outcome, - page_delta_information=page_delta_information, - api_config=api_config, - ) - res = self._browser_api_client.make_interaction(dto) - if res.status == "failed": - raise IncorrectOutcomeError( - message=res.message, - screenshot_base64=page_delta_information.page_after.screenshot_base64, - ) - return res - - return wrapper - - return decorator - - -class Element: - """ - Represents an element in the Dendrite browser environment. Wraps a Playwright Locator. - - This class provides methods for interacting with and manipulating - elements in the browser. - """ - - def __init__( - self, - dendrite_id: str, - locator: Locator, - dendrite_browser: Dendrite, - browser_api_client: BrowserAPIClient, - ): - """ - Initialize a Element. - - Args: - dendrite_id (str): The dendrite_id identifier for this element. - locator (Locator): The Playwright locator for this element. - dendrite_browser (Dendrite): The browser instance. - """ - self.dendrite_id = dendrite_id - self.locator = locator - self._dendrite_browser = dendrite_browser - self._browser_api_client = browser_api_client - - def outer_html(self): - return self.locator.evaluate("(element) => element.outerHTML") - - def screenshot(self) -> str: - """ - Take a screenshot of the element and return it as a base64-encoded string. - - Returns: - str: A base64-encoded string of the JPEG image. - Returns an empty string if the screenshot fails. - """ - image_data = self.locator.screenshot(type="jpeg", timeout=20000) - if image_data is None: - return "" - return base64.b64encode(image_data).decode() - - @perform_action("click") - def click( - self, - expected_outcome: Optional[str] = None, - wait_for_navigation: bool = True, - *args, - **kwargs, - ) -> InteractionResponse: - """ - Click the element. - - Args: - expected_outcome (Optional[str]): The expected outcome of the click action. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - - Returns: - InteractionResponse: The response from the interaction. - """ - timeout = kwargs.pop("timeout", 2000) - force = kwargs.pop("force", False) - page = self._dendrite_browser.get_active_page() - navigation_tracker = NavigationTracker(page) - navigation_tracker.start_nav_tracking() - try: - self.locator.click(*args, timeout=timeout, force=force, **kwargs) - except Exception as e: - try: - self.locator.click(*args, timeout=2000, force=True, **kwargs) - except Exception as e: - self.locator.dispatch_event("click", timeout=2000) - if wait_for_navigation: - has_navigated = navigation_tracker.has_navigated_since_start() - if has_navigated: - try: - start_time = time.time() - page.playwright_page.wait_for_load_state("load", timeout=2000) - wait_duration = time.time() - start_time - except Exception as e: - pass - return InteractionResponse(status="success", message="") - - @perform_action("fill") - def fill( - self, value: str, expected_outcome: Optional[str] = None, *args, **kwargs - ) -> InteractionResponse: - """ - Fill the element with a value. If the element itself is not fillable, - it attempts to find and fill a fillable child element. - - Args: - value (str): The value to fill the element with. - expected_outcome (Optional[str]): The expected outcome of the fill action. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - - Returns: - InteractionResponse: The response from the interaction. - """ - timeout = kwargs.pop("timeout", 2000) - try: - self.locator.fill(value, *args, timeout=timeout, **kwargs) - except Exception as e: - fillable_child = self.locator.locator( - 'input, textarea, [contenteditable="true"]' - ).first - fillable_child.fill(value, *args, timeout=timeout, **kwargs) - return InteractionResponse(status="success", message="") - - @perform_action("hover") - def hover( - self, expected_outcome: Optional[str] = None, *args, **kwargs - ) -> InteractionResponse: - """ - Hover over the element. - All additional arguments are passed to the Playwright fill method. - - Args: - expected_outcome (Optional[str]): The expected outcome of the hover action. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - - Returns: - InteractionResponse: The response from the interaction. - """ - timeout = kwargs.pop("timeout", 2000) - self.locator.hover(*args, timeout=timeout, **kwargs) - return InteractionResponse(status="success", message="") - - def focus(self): - """ - Focus on the element. - """ - self.locator.focus() - - def highlight(self): - """ - Highlights the element. This is a convenience method for debugging purposes. - """ - self.locator.highlight() - - def _wait_for_page_changes(self, old_url: str, timeout: float = 2000): - """ - Wait for page changes after an action. - - Args: - old_url (str): The URL before the action. - timeout (float): The maximum time (in milliseconds) to wait for changes. - - Returns: - bool: True if the page changed, False otherwise. - """ - timeout_in_seconds = timeout / 1000 - start_time = time.time() - while time.time() - start_time <= timeout_in_seconds: - page = self._dendrite_browser.get_active_page() - if page.url != old_url: - return True - time.sleep(0.1) - return False diff --git a/dendrite/browser/sync_api/_core/dendrite_page.py b/dendrite/browser/sync_api/_core/dendrite_page.py deleted file mode 100644 index 947021b..0000000 --- a/dendrite/browser/sync_api/_core/dendrite_page.py +++ /dev/null @@ -1,375 +0,0 @@ -import re -import time -import pathlib -import time -from typing import TYPE_CHECKING, Any, List, Literal, Optional, Sequence, Union -from bs4 import BeautifulSoup, Tag -from loguru import logger -from playwright.sync_api import FrameLocator, Keyboard, Download, FilePayload -from dendrite.browser.sync_api._api.browser_api_client import BrowserAPIClient -from dendrite.browser.sync_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.sync_api._core.dendrite_element import Element -from dendrite.browser.sync_api._core.mixin.ask import AskMixin -from dendrite.browser.sync_api._core.mixin.click import ClickMixin -from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin -from dendrite.browser.sync_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.sync_api._core.mixin.keyboard import KeyboardMixin -from dendrite.browser.sync_api._core.mixin.markdown import MarkdownMixin -from dendrite.browser.sync_api._core.mixin.wait_for import WaitForMixin -from dendrite.browser.sync_api._core.models.page_information import PageInformation - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser.sync_api._core._managers.screenshot_manager import ScreenshotManager -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.sync_api._core._utils import expand_iframes - - -class Page( - MarkdownMixin, - ExtractionMixin, - WaitForMixin, - AskMixin, - FillFieldsMixin, - ClickMixin, - KeyboardMixin, - GetElementMixin, -): - """ - Represents a page in the Dendrite browser environment. - - This class provides methods for interacting with and manipulating - pages in the browser. - """ - - def __init__( - self, - page: PlaywrightPage, - dendrite_browser: "Dendrite", - browser_api_client: "BrowserAPIClient", - ): - self.playwright_page = page - self.screenshot_manager = ScreenshotManager(page) - self.dendrite_browser = dendrite_browser - self._browser_api_client = browser_api_client - self._last_main_frame_url = page.url - self._last_frame_navigated_timestamp = time.time() - self.playwright_page.on("framenavigated", self._on_frame_navigated) - - def _on_frame_navigated(self, frame): - if frame is self.playwright_page.main_frame: - self._last_main_frame_url = frame.url - self._last_frame_navigated_timestamp = time.time() - - @property - def url(self): - """ - Get the current URL of the page. - - Returns: - str: The current URL. - """ - return self.playwright_page.url - - @property - def keyboard(self) -> Keyboard: - """ - Get the keyboard object for the page. - - Returns: - Keyboard: The Playwright Keyboard object. - """ - return self.playwright_page.keyboard - - def _get_page(self) -> "Page": - return self - - def _get_dendrite_browser(self) -> "Dendrite": - return self.dendrite_browser - - def _get_browser_api_client(self) -> BrowserAPIClient: - return self._browser_api_client - - def goto( - self, - url: str, - timeout: Optional[float] = 30000, - wait_until: Optional[ - Literal["commit", "domcontentloaded", "load", "networkidle"] - ] = "load", - ) -> None: - """ - Navigate to a URL. - - Args: - url (str): The URL to navigate to. If no protocol is specified, 'https://' will be added. - timeout (Optional[float]): Maximum navigation time in milliseconds. - wait_until (Optional[Literal["commit", "domcontentloaded", "load", "networkidle"]]): - When to consider navigation succeeded. - """ - if not re.match("^\\w+://", url): - url = f"https://{url}" - self.playwright_page.goto(url, timeout=timeout, wait_until=wait_until) - - def get_download(self, timeout: float = 30000) -> Download: - """ - Retrieves the download event associated with. - - Args: - timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the download to complete. Defaults to 30. - - Returns: - The downloaded file data. - """ - return self.dendrite_browser._get_download(self.playwright_page, timeout) - - def _get_context(self, element: Any) -> Union[PlaywrightPage, FrameLocator]: - """ - Gets the correct context to be able to interact with an element on a different frame. - - e.g. if the element is inside an iframe, - the context will be the frame locator for that iframe. - - Args: - element (Any): The element to get the context for. - - Returns: - Union[Page, FrameLocator]: The context for the element. - """ - context = self.playwright_page - if isinstance(element, Tag): - full_path = element.get("iframe-path") - if full_path: - full_path = full_path[0] if isinstance(full_path, list) else full_path - for path in full_path.split("|"): - context = context.frame_locator(f"xpath=//iframe[@d-id='{path}']") - return context - - def scroll_to_bottom( - self, - timeout: float = 30000, - scroll_increment: int = 1000, - no_progress_limit: int = 3, - ) -> None: - """ - Scrolls to the bottom of the page until no more progress is made or a timeout occurs. - - Args: - timeout (float, optional): The maximum amount of time (in milliseconds) to continue scrolling. Defaults to 30000. - scroll_increment (int, optional): The number of pixels to scroll in each step. Defaults to 1000. - no_progress_limit (int, optional): The number of consecutive attempts with no progress before stopping. Defaults to 3. - - Returns: - None - """ - start_time = time.time() - last_scroll_position = 0 - no_progress_count = 0 - while True: - current_scroll_position = self.playwright_page.evaluate("window.scrollY") - scroll_height = self.playwright_page.evaluate("document.body.scrollHeight") - self.playwright_page.evaluate( - f"window.scrollTo(0, {current_scroll_position + scroll_increment})" - ) - if ( - self.playwright_page.viewport_size - and current_scroll_position - + self.playwright_page.viewport_size["height"] - >= scroll_height - ): - break - if current_scroll_position > last_scroll_position: - no_progress_count = 0 - else: - no_progress_count += 1 - if no_progress_count >= no_progress_limit: - break - if time.time() - start_time > timeout * 0.001: - break - last_scroll_position = current_scroll_position - time.sleep(0.1) - - def close(self) -> None: - """ - Closes the current page. - - Returns: - None - """ - self.playwright_page.close() - - def get_page_information(self, include_screenshot: bool = True) -> PageInformation: - """ - Retrieves information about the current page, including the URL, raw HTML, and a screenshot. - - Returns: - PageInformation: An object containing the page's URL, raw HTML, and a screenshot in base64 format. - """ - if include_screenshot: - base64 = self.screenshot_manager.take_full_page_screenshot() - else: - base64 = "No screenshot available" - soup = self._get_soup() - return PageInformation( - url=self.playwright_page.url, - raw_html=str(soup), - screenshot_base64=base64, - time_since_frame_navigated=self.get_time_since_last_frame_navigated(), - ) - - def _generate_dendrite_ids(self): - """ - Attempts to generate Dendrite IDs in the DOM by executing a script. - - This method will attempt to generate the Dendrite IDs up to 3 times. If all attempts fail, - an exception is raised. - - Raises: - Exception: If the Dendrite IDs could not be generated after 3 attempts. - """ - tries = 0 - while tries < 3: - try: - self.playwright_page.evaluate(GENERATE_DENDRITE_IDS_SCRIPT) - return - except Exception as e: - self.playwright_page.wait_for_load_state(state="load", timeout=3000) - logger.debug( - f"Failed to generate dendrite IDs: {e}, attempt {tries + 1}/3" - ) - tries += 1 - raise DendriteException("Failed to add d-ids to DOM.") - - def scroll_through_entire_page(self) -> None: - """ - Scrolls through the entire page. - - Returns: - None - """ - self.scroll_to_bottom() - - def upload_files( - self, - files: Union[ - str, - pathlib.Path, - FilePayload, - Sequence[Union[str, pathlib.Path]], - Sequence[FilePayload], - ], - timeout: float = 30000, - ) -> None: - """ - Uploads files to the page using a file chooser. - - Args: - files (Union[str, pathlib.Path, FilePayload, Sequence[Union[str, pathlib.Path]], Sequence[FilePayload]]): The file(s) to be uploaded. - This can be a file path, a `FilePayload` object, or a sequence of file paths or `FilePayload` objects. - timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the file chooser to be ready. Defaults to 30. - - Returns: - None - """ - file_chooser = self.dendrite_browser._get_filechooser( - self.playwright_page, timeout - ) - file_chooser.set_files(files) - - def get_content(self): - """ - Retrieves the content of the current page. - - Returns: - str: The HTML content of the current page. - """ - return self.playwright_page.content() - - def _get_soup(self) -> BeautifulSoup: - """ - Retrieves the page source as a BeautifulSoup object, with an option to exclude hidden elements. - Generates Dendrite IDs in the DOM and expands iframes. - - Returns: - BeautifulSoup: The parsed HTML of the current page. - """ - self._generate_dendrite_ids() - page_source = self.playwright_page.content() - soup = BeautifulSoup(page_source, "lxml") - self._expand_iframes(soup) - self._previous_soup = soup - return soup - - def _get_previous_soup(self) -> BeautifulSoup: - """ - Retrieves the page source generated by the latest _get_soup() call as a Beautiful soup object. If it hasn't been called yet, it will call it. - """ - if self._previous_soup is None: - return self._get_soup() - return self._previous_soup - - def _expand_iframes(self, page_source: BeautifulSoup): - """ - Expands iframes in the given page source to make their content accessible. - - Args: - page_source (BeautifulSoup): The parsed HTML content of the page. - - Returns: - None - """ - expand_iframes(self.playwright_page, page_source) - - def _get_all_elements_from_selector(self, selector: str) -> List[Element]: - dendrite_elements: List[Element] = [] - soup = self._get_soup() - elements = soup.select(selector) - for element in elements: - frame = self._get_context(element) - d_id = element.get("d-id", "") - locator = frame.locator(f"xpath=//*[@d-id='{d_id}']") - if not d_id: - continue - if isinstance(d_id, list): - d_id = d_id[0] - dendrite_elements.append( - Element(d_id, locator, self.dendrite_browser, self._browser_api_client) - ) - return dendrite_elements - - def _dump_html(self, path: str) -> None: - """ - Saves the current page's HTML content to a file. - - Args: - path (str): The file path where the HTML content should be saved. - - Returns: - None - """ - with open(path, "w") as f: - f.write(self.playwright_page.content()) - - def get_time_since_last_frame_navigated(self) -> float: - """ - Get the time elapsed since the last URL change. - - Returns: - float: The number of seconds elapsed since the last URL change. - """ - return time.time() - self._last_frame_navigated_timestamp - - def check_if_renavigated(self, initial_url: str, wait_time: float = 0.1) -> bool: - """ - Waits for a short period and checks if a main frame navigation has occurred. - - Args: - wait_time (float): The time to wait in seconds. Defaults to 0.1 seconds. - - Returns: - bool: True if a main frame navigation occurred, False otherwise. - """ - time.sleep(wait_time) - return self._last_main_frame_url != initial_url diff --git a/dendrite/browser/sync_api/_core/mixin/ask.py b/dendrite/browser/sync_api/_core/mixin/ask.py deleted file mode 100644 index ff1da2f..0000000 --- a/dendrite/browser/sync_api/_core/mixin/ask.py +++ /dev/null @@ -1,191 +0,0 @@ -import time -import time -from typing import Optional, Type, overload -from loguru import logger -from dendrite.browser.sync_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.browser.sync_api._core._type_spec import ( - JsonSchema, - PydanticModel, - TypeSpec, - convert_to_type_spec, - to_json_schema, -) -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException - -TIMEOUT_INTERVAL = [150, 450, 1000] - - -class AskMixin(DendritePageProtocol): - - @overload - def ask(self, prompt: str, type_spec: Type[str]) -> str: - """ - Asks a question about the current page and expects a response of type `str`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[str]): The expected return type, which is `str`. - - Returns: - AskPageResponse[str]: The response object containing the result of type `str`. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[bool]) -> bool: - """ - Asks a question about the current page and expects a responseof type `bool`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[bool]): The expected return type, which is `bool`. - - Returns: - AskPageResponse[bool]: The response object containing the result of type `bool`. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[int]) -> int: - """ - Asks a question about the current page and expects a response of type `int`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[int]): The expected return type, which is `int`. - - Returns: - AskPageResponse[int]: The response object containing the result of type `int`. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[float]) -> float: - """ - Asks a question about the current page and expects a response of type `float`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[float]): The expected return type, which is `float`. - - Returns: - AskPageResponse[float]: The response object containing the result of type `float`. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[PydanticModel]) -> PydanticModel: - """ - Asks a question about the current page and expects a response of a custom `PydanticModel`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[PydanticModel]): The expected return type, which is a `PydanticModel`. - - Returns: - AskPageResponse[PydanticModel]: The response object containing the result of the specified Pydantic model type. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[JsonSchema]) -> JsonSchema: - """ - Asks a question about the current page and expects a response conforming to a `JsonSchema`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[JsonSchema]): The expected return type, which is a `JsonSchema`. - - Returns: - AskPageResponse[JsonSchema]: The response object containing the result conforming to the specified JSON schema. - """ - - @overload - def ask(self, prompt: str, type_spec: None = None) -> JsonSchema: - """ - Asks a question without specifying a type and expects a response conforming to a default `JsonSchema`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (None, optional): The expected return type, which is `None` by default. - - Returns: - AskPageResponse[JsonSchema]: The response object containing the result conforming to the default JSON schema. - """ - - def ask( - self, prompt: str, type_spec: Optional[TypeSpec] = None, timeout: int = 15000 - ) -> TypeSpec: - """ - Asks a question and processes the response based on the specified type. - - This method sends a request to ask a question with the specified prompt and processes the response. - If a type specification is provided, the response is converted to the specified type. In case of failure, - a DendriteException is raised with relevant details. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Optional[TypeSpec], optional): The expected return type, which can be a type or a schema. Defaults to None. - - Returns: - AskPageResponse[Any]: The response object containing the result, converted to the specified type if provided. - - Raises: - DendriteException: If the request fails, the exception includes the failure message and a screenshot. - """ - api_config = self._get_dendrite_browser().api_config - start_time = time.time() - attempt_start = start_time - attempt = -1 - while True: - attempt += 1 - current_timeout = ( - TIMEOUT_INTERVAL[attempt] - if len(TIMEOUT_INTERVAL) > attempt - else TIMEOUT_INTERVAL[-1] * 1.75 - ) - elapsed_time = time.time() - start_time - remaining_time = timeout * 0.001 - elapsed_time - if remaining_time <= 0: - logger.warning( - f"Timeout reached for '{prompt}' after {attempt + 1} attempts" - ) - break - prev_attempt_time = time.time() - attempt_start - sleep_time = min( - max(current_timeout * 0.001 - prev_attempt_time, 0), remaining_time - ) - logger.debug(f"Waiting for {sleep_time} seconds before retrying") - time.sleep(sleep_time) - attempt_start = time.time() - logger.info(f"Asking '{prompt}' | Attempt {attempt + 1}") - page = self._get_page() - page_information = page.get_page_information() - schema = to_json_schema(type_spec) if type_spec else None - if elapsed_time < 5: - time_prompt = f"This page was loaded {elapsed_time} seconds ago, so it might still be loading. If the page is still loading, return failed status." - else: - time_prompt = "" - entire_prompt = prompt + time_prompt - dto = AskPageDTO( - page_information=page_information, - api_config=api_config, - prompt=entire_prompt, - return_schema=schema, - ) - try: - res = self._get_browser_api_client().ask_page(dto) - logger.debug(f"Got response in {time.time() - attempt_start} seconds") - if res.status == "error": - logger.warning( - f"Error response on attempt {attempt + 1}: {res.return_data}" - ) - continue - converted_res = res.return_data - if type_spec is not None: - converted_res = convert_to_type_spec(type_spec, res.return_data) - return converted_res - except Exception as e: - logger.error(f"Exception occurred on attempt {attempt + 1}: {str(e)}") - if attempt == len(TIMEOUT_INTERVAL) - 1: - raise - raise DendriteException( - message=f"Failed to get response for '{prompt}' after {attempt + 1} attempts", - screenshot_base64=page_information.screenshot_base64, - ) diff --git a/dendrite/browser/sync_api/_core/mixin/click.py b/dendrite/browser/sync_api/_core/mixin/click.py deleted file mode 100644 index 80fc071..0000000 --- a/dendrite/browser/sync_api/_core/mixin/click.py +++ /dev/null @@ -1,57 +0,0 @@ -import time -from typing import Any, Optional -from dendrite.browser.sync_api._api.response.interaction_response import InteractionResponse -from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException - - -class ClickMixin(GetElementMixin, DendritePageProtocol): - - def click( - self, - prompt: str, - expected_outcome: Optional[str] = None, - use_cache: bool = True, - timeout: int = 15000, - force: bool = False, - *args, - **kwargs, - ) -> InteractionResponse: - """ - Clicks an element on the page based on the provided prompt. - - This method combines the functionality of get_element and click, - allowing for a more concise way to interact with elements on the page. - - Args: - prompt (str): The prompt describing the element to be clicked. - expected_outcome (Optional[str]): The expected outcome of the click action. - use_cache (bool, optional): Whether to use cached results for element retrieval. Defaults to True. - timeout (int, optional): The timeout (in milliseconds) for the click operation. Defaults to 15000. - force (bool, optional): Whether to force the click operation. Defaults to False. - *args: Additional positional arguments for the click operation. - **kwargs: Additional keyword arguments for the click operation. - - Returns: - InteractionResponse: The response from the interaction. - - Raises: - DendriteException: If no suitable element is found or if the click operation fails. - """ - augmented_prompt = prompt + "\n\nThe element should be clickable." - element = self.get_element( - augmented_prompt, use_cache=use_cache, timeout=timeout - ) - if not element: - raise DendriteException( - message=f"No element found with the prompt: {prompt}", - screenshot_base64="", - ) - return element.click( - *args, - expected_outcome=expected_outcome, - timeout=timeout, - force=force, - **kwargs, - ) diff --git a/dendrite/browser/sync_api/_core/mixin/extract.py b/dendrite/browser/sync_api/_core/mixin/extract.py deleted file mode 100644 index 108806f..0000000 --- a/dendrite/browser/sync_api/_core/mixin/extract.py +++ /dev/null @@ -1,232 +0,0 @@ -import time -import time -from typing import Any, Optional, Type, overload, List -from dendrite.browser.sync_api._api.dto.extract_dto import ExtractDTO -from dendrite.browser.sync_api._api.response.cache_extract_response import CacheExtractResponse -from dendrite.browser.sync_api._api.response.extract_response import ExtractResponse -from dendrite.browser.sync_api._core._type_spec import ( - JsonSchema, - PydanticModel, - TypeSpec, - convert_to_type_spec, - to_json_schema, -) -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser.sync_api._core._managers.navigation_tracker import NavigationTracker -from loguru import logger - -CACHE_TIMEOUT = 5 - - -class ExtractionMixin(DendritePageProtocol): - """ - Mixin that provides extraction functionality for web pages. - - This mixin provides various `extract` methods that allow extracting - different types of data (e.g., bool, int, float, string, Pydantic models, etc.) - from a web page based on a given prompt. - """ - - @overload - def extract( - self, - prompt: str, - type_spec: Type[bool], - use_cache: bool = True, - timeout: int = 180, - ) -> bool: ... - - @overload - def extract( - self, - prompt: str, - type_spec: Type[int], - use_cache: bool = True, - timeout: int = 180, - ) -> int: ... - - @overload - def extract( - self, - prompt: str, - type_spec: Type[float], - use_cache: bool = True, - timeout: int = 180, - ) -> float: ... - - @overload - def extract( - self, - prompt: str, - type_spec: Type[str], - use_cache: bool = True, - timeout: int = 180, - ) -> str: ... - - @overload - def extract( - self, - prompt: Optional[str], - type_spec: Type[PydanticModel], - use_cache: bool = True, - timeout: int = 180, - ) -> PydanticModel: ... - - @overload - def extract( - self, - prompt: Optional[str], - type_spec: JsonSchema, - use_cache: bool = True, - timeout: int = 180, - ) -> JsonSchema: ... - - @overload - def extract( - self, - prompt: str, - type_spec: None = None, - use_cache: bool = True, - timeout: int = 180, - ) -> Any: ... - - def extract( - self, - prompt: Optional[str], - type_spec: Optional[TypeSpec] = None, - use_cache: bool = True, - timeout: int = 180, - ) -> TypeSpec: - """ - Extract data from a web page based on a prompt and optional type specification. - Args: - prompt (Optional[str]): The prompt to describe the information to extract. - type_spec (Optional[TypeSpec], optional): The type specification for the extracted data. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached scripts before falling back to the - extraction agent for the remaining time that will attempt to generate a new script. Defaults to 15000 (15 seconds). - - Returns: - ExtractResponse: The extracted data wrapped in a ExtractResponse object. - Raises: - TimeoutError: If the extraction process exceeds the specified timeout. - """ - logger.info(f"Starting extraction with prompt: {prompt}") - json_schema = None - if type_spec: - json_schema = to_json_schema(type_spec) - logger.debug(f"Type specification converted to JSON schema: {json_schema}") - if prompt is None: - prompt = "" - start_time = time.time() - page = self._get_page() - navigation_tracker = NavigationTracker(page) - navigation_tracker.start_nav_tracking() - if use_cache: - cache_available = check_if_extract_cache_available( - self, prompt, json_schema - ) - if cache_available: - logger.info("Cache available, attempting to use cached extraction") - result = attempt_extraction_with_backoff( - self, - prompt, - json_schema, - remaining_timeout=CACHE_TIMEOUT, - only_use_cache=True, - ) - if result: - return convert_and_return_result(result, type_spec) - logger.info( - "Using extraction agent to perform extraction, since no cache was found or failed." - ) - result = attempt_extraction_with_backoff( - self, - prompt, - json_schema, - remaining_timeout=timeout - (time.time() - start_time), - only_use_cache=False, - ) - if result: - return convert_and_return_result(result, type_spec) - logger.error(f"Extraction failed after {time.time() - start_time:.2f} seconds") - return None - - -def check_if_extract_cache_available( - obj: DendritePageProtocol, prompt: str, json_schema: Optional[JsonSchema] -) -> bool: - page = obj._get_page() - page_information = page.get_page_information(include_screenshot=False) - dto = ExtractDTO( - page_information=page_information, - api_config=obj._get_dendrite_browser().api_config, - prompt=prompt, - return_data_json_schema=json_schema, - ) - cache_response: CacheExtractResponse = ( - obj._get_browser_api_client().check_extract_cache(dto) - ) - return cache_response.exists - - -def attempt_extraction_with_backoff( - obj: DendritePageProtocol, - prompt: str, - json_schema: Optional[JsonSchema], - remaining_timeout: float = 180.0, - only_use_cache: bool = False, -) -> Optional[ExtractResponse]: - TIMEOUT_INTERVAL: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0] - total_elapsed_time = 0 - start_time = time.time() - for current_timeout in TIMEOUT_INTERVAL: - if total_elapsed_time >= remaining_timeout: - logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") - return None - request_start_time = time.time() - page = obj._get_page() - page_information = page.get_page_information( - include_screenshot=not only_use_cache - ) - extract_dto = ExtractDTO( - page_information=page_information, - api_config=obj._get_dendrite_browser().api_config, - prompt=prompt, - return_data_json_schema=json_schema, - use_screenshot=True, - use_cache=only_use_cache, - force_use_cache=only_use_cache, - ) - res = obj._get_browser_api_client().extract(extract_dto) - request_duration = time.time() - request_start_time - if res.status == "impossible": - logger.error(f"Impossible to extract data. Reason: {res.message}") - return None - if res.status == "success": - logger.success( - f"Extraction successful: '{res.message}'\nUsed cache: {res.used_cache}\nUsed script:\n\n{res.created_script}" - ) - return res - sleep_duration = max(0, current_timeout - request_duration) - logger.info( - f"Extraction attempt failed. Status: {res.status}\nMessage: {res.message}\nSleeping for {sleep_duration:.2f} seconds" - ) - time.sleep(sleep_duration) - total_elapsed_time = time.time() - start_time - logger.error( - f"All extraction attempts failed after {total_elapsed_time:.2f} seconds" - ) - return None - - -def convert_and_return_result( - res: ExtractResponse, type_spec: Optional[TypeSpec] -) -> TypeSpec: - converted_res = res.return_data - if type_spec is not None: - logger.debug("Converting extraction result to specified type") - converted_res = convert_to_type_spec(type_spec, res.return_data) - logger.info("Extraction process completed successfully") - return converted_res diff --git a/dendrite/browser/sync_api/_core/mixin/fill_fields.py b/dendrite/browser/sync_api/_core/mixin/fill_fields.py deleted file mode 100644 index 885a54c..0000000 --- a/dendrite/browser/sync_api/_core/mixin/fill_fields.py +++ /dev/null @@ -1,76 +0,0 @@ -import time -from typing import Any, Dict, Optional -from dendrite.browser.sync_api._api.response.interaction_response import InteractionResponse -from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException - - -class FillFieldsMixin(GetElementMixin, DendritePageProtocol): - - def fill_fields(self, fields: Dict[str, Any]): - """ - Fills multiple fields on the page with the provided values. - - This method iterates through the given dictionary of fields and their corresponding values, - making a separate fill request for each key-value pair. - - Args: - fields (Dict[str, Any]): A dictionary where each key is a field identifier (e.g., a prompt or selector) - and each value is the content to fill in that field. - - Returns: - None - - Note: - This method will make multiple fill requests, one for each key in the 'fields' dictionary. - """ - for field, value in fields.items(): - prompt = f"I'll be filling in text in several fields with these keys: {fields.keys()} in this page. Get the field best described as '{field}'. I want to fill it with a '{type(value)}' type value." - self.fill(prompt, value) - time.sleep(0.5) - - def fill( - self, - prompt: str, - value: str, - expected_outcome: Optional[str] = None, - use_cache: bool = True, - timeout: int = 15000, - *args, - kwargs={}, - ) -> InteractionResponse: - """ - Fills an element on the page with the provided value based on the given prompt. - - This method combines the functionality of get_element and fill, - allowing for a more concise way to interact with elements on the page. - - Args: - prompt (str): The prompt describing the element to be filled. - value (str): The value to fill the element with. - expected_outcome (Optional[str]): The expected outcome of the fill action. - use_cache (bool, optional): Whether to use cached results for element retrieval. Defaults to True. - max_retries (int, optional): The maximum number of retry attempts for element retrieval. Defaults to 3. - timeout (int, optional): The timeout (in milliseconds) for the fill operation. Defaults to 15000. - *args: Additional positional arguments for the fill operation. - kwargs: Additional keyword arguments for the fill operation. - - Returns: - InteractionResponse: The response from the interaction. - - Raises: - DendriteException: If no suitable element is found or if the fill operation fails. - """ - augmented_prompt = prompt + "\n\nMake sure the element can be filled with text." - element = self.get_element( - augmented_prompt, use_cache=use_cache, timeout=timeout - ) - if not element: - raise DendriteException( - message=f"No element found with the prompt: {prompt}", - screenshot_base64="", - ) - return element.fill( - value, *args, expected_outcome=expected_outcome, timeout=timeout, **kwargs - ) diff --git a/dendrite/browser/sync_api/_core/mixin/get_element.py b/dendrite/browser/sync_api/_core/mixin/get_element.py deleted file mode 100644 index 3fe7511..0000000 --- a/dendrite/browser/sync_api/_core/mixin/get_element.py +++ /dev/null @@ -1,302 +0,0 @@ -import time -import time -from typing import Dict, List, Literal, Optional, Union, overload -from loguru import logger -from dendrite.browser.sync_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.browser.sync_api._api.response.get_element_response import GetElementResponse -from dendrite.browser.sync_api._api.dto.get_elements_dto import CheckSelectorCacheDTO -from dendrite.browser.sync_api._core._utils import get_elements_from_selectors_soup -from dendrite.browser.sync_api._core.dendrite_element import Element -from dendrite.browser.sync_api._core.models.response import ElementsResponse -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser.sync_api._core.models.api_config import APIConfig - -CACHE_TIMEOUT = 5 - - -class GetElementMixin(DendritePageProtocol): - - @overload - def get_elements( - self, - prompt_or_elements: str, - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> List[Element]: - """ - Retrieves a list of Dendrite elements based on a string prompt. - - Args: - prompt_or_elements (str): The prompt describing the elements to be retrieved. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - List[Element]: A list of Dendrite elements found on the page. - """ - - @overload - def get_elements( - self, - prompt_or_elements: Dict[str, str], - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> ElementsResponse: - """ - Retrieves Dendrite elements based on a dictionary. - - Args: - prompt_or_elements (Dict[str, str]): A dictionary where keys are field names and values are prompts describing the elements to be retrieved. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - ElementsResponse: A response object containing the retrieved elements with attributes matching the keys in the dict. - """ - - def get_elements( - self, - prompt_or_elements: Union[str, Dict[str, str]], - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> Union[List[Element], ElementsResponse]: - """ - Retrieves Dendrite elements based on either a string prompt or a dictionary of prompts. - - This method determines the type of the input (string or dictionary) and retrieves the appropriate elements. - If the input is a string, it fetches a list of elements. If the input is a dictionary, it fetches elements for each key-value pair. - - Args: - prompt_or_elements (Union[str, Dict[str, str]]): The prompt or dictionary of prompts for element retrieval. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - Union[List[Element], ElementsResponse]: A list of elements or a response object containing the retrieved elements. - - Raises: - ValueError: If the input is neither a string nor a dictionary. - """ - return self._get_element( - prompt_or_elements, - only_one=False, - use_cache=use_cache, - timeout=timeout / 1000, - ) - - def get_element( - self, prompt: str, use_cache=True, timeout=15000 - ) -> Optional[Element]: - """ - Retrieves a single Dendrite element based on the provided prompt. - - Args: - prompt (str): The prompt describing the element to be retrieved. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - - Returns: - Element: The retrieved element. - """ - logger.info(f"Getting element for prompt: {prompt}") - return self._get_element( - prompt, only_one=True, use_cache=use_cache, timeout=timeout / 1000 - ) - - @overload - def _get_element( - self, prompt_or_elements: str, only_one: Literal[True], use_cache: bool, timeout - ) -> Optional[Element]: - """ - Retrieves a single Dendrite element based on the provided prompt. - - Args: - prompt (Union[str, Dict[str, str]]): The prompt describing the element to be retrieved. - only_one (Literal[True]): Indicates that only one element should be retrieved. - use_cache (bool): Whether to use cached results. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - - Returns: - Element: The retrieved element. - """ - - @overload - def _get_element( - self, - prompt_or_elements: Union[str, Dict[str, str]], - only_one: Literal[False], - use_cache: bool, - timeout, - ) -> Union[List[Element], ElementsResponse]: - """ - Retrieves a list of Dendrite elements based on the provided prompt. - - Args: - prompt (str): The prompt describing the elements to be retrieved. - only_one (Literal[False]): Indicates that multiple elements should be retrieved. - use_cache (bool): Whether to use cached results. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - - Returns: - List[Element]: A list of retrieved elements. - """ - - def _get_element( - self, - prompt_or_elements: Union[str, Dict[str, str]], - only_one: bool, - use_cache: bool, - timeout: float, - ) -> Union[Optional[Element], List[Element], ElementsResponse]: - """ - Retrieves Dendrite elements based on the provided prompt, either a single element or a list of elements. - - This method sends a request with the prompt and retrieves the elements based on the `only_one` flag. - - Args: - prompt_or_elements (Union[str, Dict[str, str]]): The prompt or dictionary of prompts for element retrieval. - only_one (bool): Whether to retrieve only one element or a list of elements. - use_cache (bool): Whether to use cached results. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - - Returns: - Union[Element, List[Element], ElementsResponse]: The retrieved element, list of elements, or response object. - """ - api_config = self._get_dendrite_browser().api_config - start_time = time.time() - page = self._get_page() - cache_available = test_if_cache_available(self, prompt_or_elements, page.url) - if cache_available and use_cache == True: - logger.info(f"Cache available, attempting to use cached selectors") - res = attempt_with_backoff( - self, - prompt_or_elements, - only_one, - api_config, - remaining_timeout=CACHE_TIMEOUT, - only_use_cache=True, - ) - if res: - return res - else: - logger.debug( - f"After attempting to use cached selectors several times without success, let's find the elements using the find element agent." - ) - logger.info( - "Proceeding to use the find element agent to find the requested elements." - ) - res = attempt_with_backoff( - self, - prompt_or_elements, - only_one, - api_config, - remaining_timeout=timeout - (time.time() - start_time), - only_use_cache=False, - ) - if res: - return res - logger.error( - f"Failed to retrieve elements within the specified timeout of {timeout} seconds" - ) - return None - - -def test_if_cache_available( - obj: DendritePageProtocol, prompt_or_elements: Union[str, Dict[str, str]], url: str -) -> bool: - dto = CheckSelectorCacheDTO(url=url, prompt=prompt_or_elements) - cache_available = obj._get_browser_api_client().check_selector_cache(dto) - return cache_available.exists - - -def attempt_with_backoff( - obj: DendritePageProtocol, - prompt_or_elements: Union[str, Dict[str, str]], - only_one: bool, - api_config: APIConfig, - remaining_timeout: float, - only_use_cache: bool = False, -) -> Union[Optional[Element], List[Element], ElementsResponse]: - TIMEOUT_INTERVAL: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0] - total_elapsed_time = 0 - start_time = time.time() - for current_timeout in TIMEOUT_INTERVAL: - if total_elapsed_time >= remaining_timeout: - logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") - return None - request_start_time = time.time() - page = obj._get_page() - page_information = page.get_page_information( - include_screenshot=not only_use_cache - ) - dto = GetElementsDTO( - page_information=page_information, - prompt=prompt_or_elements, - api_config=api_config, - use_cache=only_use_cache, - only_one=only_one, - force_use_cache=only_use_cache, - ) - res = obj._get_browser_api_client().get_interactions_selector(dto) - request_duration = time.time() - request_start_time - if res.status == "impossible": - logger.error( - f"Impossible to get elements for '{prompt_or_elements}'. Reason: {res.message}" - ) - return None - if res.status == "success": - response = get_elements_from_selectors_soup( - page, page._get_previous_soup(), res, only_one - ) - if response: - return response - sleep_duration = max(0, current_timeout - request_duration) - logger.info( - f"Failed to get elements for prompt:\n\n'{prompt_or_elements}'\n\nStatus: {res.status}\n\nMessage: {res.message}\n\nSleeping for {sleep_duration:.2f} seconds" - ) - time.sleep(sleep_duration) - total_elapsed_time = time.time() - start_time - logger.error(f"All attempts failed after {total_elapsed_time:.2f} seconds") - return None - - -def get_elements_from_selectors( - obj: DendritePageProtocol, res: GetElementResponse, only_one: bool -) -> Union[Optional[Element], List[Element], ElementsResponse]: - if isinstance(res.selectors, dict): - result = {} - for key, selectors in res.selectors.items(): - for selector in selectors: - page = obj._get_page() - dendrite_elements = page._get_all_elements_from_selector(selector) - if len(dendrite_elements) > 0: - result[key] = dendrite_elements[0] - break - return ElementsResponse(result) - elif isinstance(res.selectors, list): - for selector in reversed(res.selectors): - page = obj._get_page() - dendrite_elements = page._get_all_elements_from_selector(selector) - if len(dendrite_elements) > 0: - return dendrite_elements[0] if only_one else dendrite_elements - return None diff --git a/dendrite/browser/sync_api/_core/mixin/keyboard.py b/dendrite/browser/sync_api/_core/mixin/keyboard.py deleted file mode 100644 index b728770..0000000 --- a/dendrite/browser/sync_api/_core/mixin/keyboard.py +++ /dev/null @@ -1,62 +0,0 @@ -from typing import Any, Union, Literal -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException - - -class KeyboardMixin(DendritePageProtocol): - - def press( - self, - key: Union[ - str, - Literal[ - "Enter", - "Tab", - "Escape", - "Backspace", - "ArrowUp", - "ArrowDown", - "ArrowLeft", - "ArrowRight", - ], - ], - hold_shift: bool = False, - hold_ctrl: bool = False, - hold_alt: bool = False, - hold_cmd: bool = False, - ): - """ - Presses a keyboard key on the active page, optionally with modifier keys. - - Args: - key (Union[str, Literal["Enter", "Tab", "Escape", "Backspace", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight"]]): The main key to be pressed. - hold_shift (bool, optional): Whether to hold the Shift key. Defaults to False. - hold_ctrl (bool, optional): Whether to hold the Control key. Defaults to False. - hold_alt (bool, optional): Whether to hold the Alt key. Defaults to False. - hold_cmd (bool, optional): Whether to hold the Command key (Meta on some systems). Defaults to False. - - Returns: - Any: The result of the key press operation. - - Raises: - DendriteException: If the key press operation fails. - """ - modifiers = [] - if hold_shift: - modifiers.append("Shift") - if hold_ctrl: - modifiers.append("Control") - if hold_alt: - modifiers.append("Alt") - if hold_cmd: - modifiers.append("Meta") - if modifiers: - key = "+".join(modifiers + [key]) - try: - page = self._get_page() - page.keyboard.press(key) - except Exception as e: - raise DendriteException( - message=f"Failed to press key: {key}. Error: {str(e)}", - screenshot_base64="", - ) diff --git a/dendrite/browser/sync_api/_core/mixin/markdown.py b/dendrite/browser/sync_api/_core/mixin/markdown.py deleted file mode 100644 index 3125cff..0000000 --- a/dendrite/browser/sync_api/_core/mixin/markdown.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import Optional -from bs4 import BeautifulSoup -import re -from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from markdownify import markdownify as md - - -class MarkdownMixin(ExtractionMixin, DendritePageProtocol): - - def markdown(self, prompt: Optional[str] = None): - page = self._get_page() - page_information = page.get_page_information() - if prompt: - extract_prompt = f"Create a script that returns the HTML from one element from the DOM that best matches this requested section of the website.\n\nDescription of section: '{prompt}'\n\nWe will be converting your returned HTML to markdown, so just return ONE stringified HTML element and nothing else. It's OK if extra information is present. Example script: 'response_data = soup.find('tag', {{'attribute': 'value'}}).prettify()'" - res = self.extract(extract_prompt) - markdown_text = md(res) - cleaned_markdown = re.sub("\\n{3,}", "\n\n", markdown_text) - return cleaned_markdown - else: - markdown_text = md(page_information.raw_html) - cleaned_markdown = re.sub("\\n{3,}", "\n\n", markdown_text) - return cleaned_markdown diff --git a/dendrite/browser/sync_api/_core/mixin/screenshot.py b/dendrite/browser/sync_api/_core/mixin/screenshot.py deleted file mode 100644 index bd3fab2..0000000 --- a/dendrite/browser/sync_api/_core/mixin/screenshot.py +++ /dev/null @@ -1,20 +0,0 @@ -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol - - -class ScreenshotMixin(DendritePageProtocol): - - def screenshot(self, full_page: bool = False) -> str: - """ - Take a screenshot of the current page. - - Args: - full_page (bool, optional): If True, captures the full page. If False, captures only the viewport. Defaults to False. - - Returns: - str: A base64 encoded string of the screenshot in JPEG format. - """ - page = self._get_page() - if full_page: - return page.screenshot_manager.take_full_page_screenshot() - else: - return page.screenshot_manager.take_viewport_screenshot() diff --git a/dendrite/browser/sync_api/_core/mixin/wait_for.py b/dendrite/browser/sync_api/_core/mixin/wait_for.py deleted file mode 100644 index 56b5bfc..0000000 --- a/dendrite/browser/sync_api/_core/mixin/wait_for.py +++ /dev/null @@ -1,51 +0,0 @@ -import time -import time -from dendrite.browser.sync_api._core.mixin.ask import AskMixin -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.browser._common._exceptions.dendrite_exception import PageConditionNotMet -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from loguru import logger - - -class WaitForMixin(AskMixin, DendritePageProtocol): - - def wait_for(self, prompt: str, timeout: float = 30000): - """ - Waits for the condition specified in the prompt to become true by periodically checking the page content. - - This method attempts to retrieve the page information and evaluate whether the specified - condition (provided in the prompt) is met. It continues to retry until the total elapsed time - exceeds the specified timeout. - - Args: - prompt (str): The prompt to determine the condition to wait for on the page. - timeout (float, optional): The maximum time (in milliseconds) to wait for the condition. Defaults to 15000. - - Returns: - Any: The result of the condition evaluation if successful. - - Raises: - PageConditionNotMet: If the condition is not met within the specified timeout. - """ - start_time = time.time() - time.sleep(0.2) - while True: - elapsed_time = (time.time() - start_time) * 1000 - if elapsed_time >= timeout: - break - page = self._get_page() - page_information = page.get_page_information() - prompt_with_instruction = f"Prompt: '{prompt}'\n\nReturn a boolean that determines if the requested information or thing is available on the page. {round(page_information.time_since_frame_navigated, 2)} seconds have passed since the page first loaded." - try: - res = self.ask(prompt_with_instruction, bool) - if res: - return res - except DendriteException as e: - logger.debug(f"Attempt failed: {e.message}") - time.sleep(0.5) - page = self._get_page() - page_information = page.get_page_information() - raise PageConditionNotMet( - message=f"Failed to wait for the requested condition within the {timeout}ms timeout.", - screenshot_base64=page_information.screenshot_base64, - ) diff --git a/dendrite/browser/sync_api/_core/models/__init__.py b/dendrite/browser/sync_api/_core/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_core/models/api_config.py b/dendrite/browser/sync_api/_core/models/api_config.py deleted file mode 100644 index 756aa5b..0000000 --- a/dendrite/browser/sync_api/_core/models/api_config.py +++ /dev/null @@ -1,30 +0,0 @@ -from typing import Optional -from pydantic import BaseModel, model_validator -from dendrite.browser._common._exceptions.dendrite_exception import MissingApiKeyError - - -class APIConfig(BaseModel): - """ - Configuration model for API keys used in the Dendrite SDK. - - Attributes: - dendrite_api_key (Optional[str]): The API key for Dendrite services. - openai_api_key (Optional[str]): The API key for OpenAI services. If you wish to use your own API key, you can do so by passing it to the Dendrite. - anthropic_api_key (Optional[str]): The API key for Anthropic services. If you wish to use your own API key, you can do so by passing it to the Dendrite. - - Raises: - ValueError: If a valid dendrite_api_key is not provided. - """ - - dendrite_api_key: Optional[str] = None - openai_api_key: Optional[str] = None - anthropic_api_key: Optional[str] = None - - @model_validator(mode="before") - def _check_api_keys(cls, values): - dendrite_api_key = values.get("dendrite_api_key") - if not dendrite_api_key: - raise MissingApiKeyError( - "A valid dendrite_api_key must be provided. Make sure you have set the DENDRITE_API_KEY environment variable or passed it to the Dendrite." - ) - return values diff --git a/dendrite/browser/sync_api/_core/models/authentication.py b/dendrite/browser/sync_api/_core/models/authentication.py deleted file mode 100644 index 3c2656e..0000000 --- a/dendrite/browser/sync_api/_core/models/authentication.py +++ /dev/null @@ -1,47 +0,0 @@ -from pydantic import BaseModel -from typing import List, Literal, Optional -from typing_extensions import TypedDict - - -class Cookie(TypedDict, total=False): - name: str - value: str - domain: str - path: str - expires: float - httpOnly: bool - secure: bool - sameSite: Literal["Lax", "None", "Strict"] - - -class LocalStorageEntry(TypedDict): - name: str - value: str - - -class OriginState(TypedDict): - origin: str - localStorage: List[LocalStorageEntry] - - -class StorageState(TypedDict, total=False): - cookies: List[Cookie] - origins: List[OriginState] - - -class DomainState(BaseModel): - domain: str - storage_state: StorageState - - -class AuthSession(BaseModel): - user_agent: Optional[str] - domain_states: List[DomainState] - - def to_storage_state(self) -> StorageState: - cookies = [] - origins = [] - for domain_state in self.domain_states: - cookies.extend(domain_state.storage_state.get("cookies", [])) - origins.extend(domain_state.storage_state.get("origins", [])) - return StorageState(cookies=cookies, origins=origins) diff --git a/dendrite/browser/sync_api/_core/models/download_interface.py b/dendrite/browser/sync_api/_core/models/download_interface.py deleted file mode 100644 index 8a68843..0000000 --- a/dendrite/browser/sync_api/_core/models/download_interface.py +++ /dev/null @@ -1,20 +0,0 @@ -from abc import ABC, abstractmethod -from pathlib import Path -from typing import Any, Union -from playwright.sync_api import Download - - -class DownloadInterface(ABC, Download): - - def __init__(self, download: Download): - self._download = download - - def __getattribute__(self, name: str) -> Any: - try: - return super().__getattribute__(name) - except AttributeError: - return getattr(self._download, name) - - @abstractmethod - def save_as(self, path: Union[str, Path]) -> None: - pass diff --git a/dendrite/browser/sync_api/_core/models/page_diff_information.py b/dendrite/browser/sync_api/_core/models/page_diff_information.py deleted file mode 100644 index 0dadf97..0000000 --- a/dendrite/browser/sync_api/_core/models/page_diff_information.py +++ /dev/null @@ -1,7 +0,0 @@ -from pydantic import BaseModel -from dendrite.browser.sync_api._core.models.page_information import PageInformation - - -class PageDiffInformation(BaseModel): - page_before: PageInformation - page_after: PageInformation diff --git a/dendrite/browser/sync_api/_core/models/page_information.py b/dendrite/browser/sync_api/_core/models/page_information.py deleted file mode 100644 index 67e1909..0000000 --- a/dendrite/browser/sync_api/_core/models/page_information.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Dict, Optional -from typing_extensions import TypedDict -from pydantic import BaseModel - - -class InteractableElementInfo(TypedDict): - attrs: Optional[str] - text: Optional[str] - - -class PageInformation(BaseModel): - url: str - raw_html: str - screenshot_base64: str - time_since_frame_navigated: float diff --git a/dendrite/browser/sync_api/_core/models/response.py b/dendrite/browser/sync_api/_core/models/response.py deleted file mode 100644 index 50d9a23..0000000 --- a/dendrite/browser/sync_api/_core/models/response.py +++ /dev/null @@ -1,54 +0,0 @@ -from typing import Dict, Iterator -from dendrite.browser.sync_api._core.dendrite_element import Element - - -class ElementsResponse: - """ - ElementsResponse is a class that encapsulates a dictionary of Dendrite elements, - allowing for attribute-style access and other convenient interactions. - - This class is used to store and access the elements retrieved by the `get_elements` function. - The attributes of this class dynamically match the keys of the dictionary passed to the `get_elements` function, - allowing for direct attribute-style access to the corresponding `Element` objects. - - Attributes: - _data (Dict[str, Element]): A dictionary where keys are the names of elements and values are the corresponding `Element` objects. - - Args: - data (Dict[str, Element]): The dictionary of elements to be encapsulated by the class. - - Methods: - __getattr__(name: str) -> Element: - Allows attribute-style access to the elements in the dictionary. - - __getitem__(key: str) -> Element: - Enables dictionary-style access to the elements. - - __iter__() -> Iterator[str]: - Provides an iterator over the keys in the dictionary. - - __repr__() -> str: - Returns a string representation of the class instance. - """ - - _data: Dict[str, Element] - - def __init__(self, data: Dict[str, Element]): - self._data = data - - def __getattr__(self, name: str) -> Element: - try: - return self._data[name] - except KeyError: - raise AttributeError( - f"'{self.__class__.__name__}' object has no attribute '{name}'" - ) - - def __getitem__(self, key: str) -> Element: - return self._data[key] - - def __iter__(self) -> Iterator[str]: - return iter(self._data) - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self._data})" diff --git a/dendrite/browser/sync_api/_core/protocol/page_protocol.py b/dendrite/browser/sync_api/_core/protocol/page_protocol.py deleted file mode 100644 index 13cb990..0000000 --- a/dendrite/browser/sync_api/_core/protocol/page_protocol.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import TYPE_CHECKING, Protocol -from dendrite.browser.sync_api._api.browser_api_client import BrowserAPIClient - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_page import Page - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite - - -class DendritePageProtocol(Protocol): - """ - Protocol that specifies the required methods and attributes - for the `ExtractionMixin` to work. - """ - - def _get_dendrite_browser(self) -> "Dendrite": ... - - def _get_browser_api_client(self) -> BrowserAPIClient: ... - - def _get_page(self) -> "Page": ... diff --git a/dendrite/browser/sync_api/_dom/__init__.py b/dendrite/browser/sync_api/_dom/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_ext_impl/__init__.py b/dendrite/browser/sync_api/_ext_impl/__init__.py deleted file mode 100644 index 4d00d3c..0000000 --- a/dendrite/browser/sync_api/_ext_impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .browserbase import BrowserbaseDownload - -__all__ = ["BrowserbaseDownload"] diff --git a/dendrite/browser/sync_api/_ext_impl/browserbase/__init__.py b/dendrite/browser/sync_api/_ext_impl/browserbase/__init__.py deleted file mode 100644 index eb977c7..0000000 --- a/dendrite/browser/sync_api/_ext_impl/browserbase/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._download import BrowserbaseDownload - -__all__ = ["BrowserbaseDownload"] diff --git a/dendrite/browser/sync_api/_ext_impl/browserbase/_client.py b/dendrite/browser/sync_api/_ext_impl/browserbase/_client.py deleted file mode 100644 index 81d2f0f..0000000 --- a/dendrite/browser/sync_api/_ext_impl/browserbase/_client.py +++ /dev/null @@ -1,63 +0,0 @@ -import time -from pathlib import Path -import time -from typing import Optional, Union -import httpx -from loguru import logger -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException - - -class BrowserbaseClient: - - def __init__(self, api_key: str, project_id: str) -> None: - self.api_key = api_key - self.project_id = project_id - - def create_session(self) -> str: - logger.debug("Creating session") - "\n Creates a session using the Browserbase API.\n\n Returns:\n str: The ID of the created session.\n " - url = "https://www.browserbase.com/v1/sessions" - headers = {"Content-Type": "application/json", "x-bb-api-key": self.api_key} - json = {"projectId": self.project_id, "keepAlive": False} - response = httpx.post(url, json=json, headers=headers) - if response.status_code >= 400: - raise DendriteException(f"Failed to create session: {response.text}") - return response.json()["id"] - - def stop_session(self, session_id: str): - url = f"https://www.browserbase.com/v1/sessions/{session_id}" - headers = {"Content-Type": "application/json", "x-bb-api-key": self.api_key} - json = {"projectId": self.project_id, "status": "REQUEST_RELEASE"} - with httpx.Client() as client: - response = client.post(url, json=json, headers=headers) - return response.json() - - def connect_url(self, enable_proxy: bool, session_id: Optional[str] = None) -> str: - url = f"wss://connect.browserbase.com?apiKey={self.api_key}" - if session_id: - url += f"&sessionId={session_id}" - if enable_proxy: - url += "&enableProxy=true" - return url - - def save_downloads_on_disk( - self, session_id: str, path: Union[str, Path], retry_for_seconds: float - ): - url = f"https://www.browserbase.com/v1/sessions/{session_id}/downloads" - headers = {"x-bb-api-key": self.api_key} - file_path = Path(path) - with httpx.Client() as session: - timeout = time.time() + retry_for_seconds - while time.time() < timeout: - try: - response = session.get(url, headers=headers) - if response.status_code == 200: - array_buffer = response.read() - if len(array_buffer) > 0: - with open(file_path, "wb") as f: - f.write(array_buffer) - return - except Exception as e: - logger.debug(f"Error fetching downloads: {e}") - time.sleep(2) - logger.debug("Failed to download files within the time limit.") diff --git a/dendrite/browser/sync_api/_ext_impl/browserbase/_download.py b/dendrite/browser/sync_api/_ext_impl/browserbase/_download.py deleted file mode 100644 index 14756f9..0000000 --- a/dendrite/browser/sync_api/_ext_impl/browserbase/_download.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -import re -import shutil -from typing import Union -import zipfile -from loguru import logger -from playwright.sync_api import Download -from dendrite.browser.sync_api._core.models.download_interface import DownloadInterface -from dendrite.browser.sync_api._ext_impl.browserbase._client import BrowserbaseClient - - -class BrowserbaseDownload(DownloadInterface): - - def __init__( - self, session_id: str, download: Download, client: BrowserbaseClient - ) -> None: - super().__init__(download) - self._session_id = session_id - self._client = client - - def save_as(self, path: Union[str, Path], timeout: float = 20) -> None: - """ - Save the latest file from the downloaded ZIP archive to the specified path. - - Args: - path (Union[str, Path]): The destination file path where the latest file will be saved. - timeout (float, optional): Timeout for the save operation. Defaults to 20 seconds. - - Raises: - Exception: If no matching files are found in the ZIP archive or if the file cannot be saved. - """ - destination_path = Path(path) - source_path = self._download.path() - destination_path.parent.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(source_path, "r") as zip_ref: - file_list = zip_ref.namelist() - sorted_files = sorted(file_list, key=extract_timestamp, reverse=True) - if not sorted_files: - raise FileNotFoundError( - "No files found in the Browserbase download ZIP" - ) - latest_file = sorted_files[0] - with zip_ref.open(latest_file) as source, open( - destination_path, "wb" - ) as target: - shutil.copyfileobj(source, target) - logger.info(f"Latest file saved successfully to {destination_path}") - - -def extract_timestamp(filename): - timestamp_pattern = re.compile("-(\\d+)\\.") - match = timestamp_pattern.search(filename) - return int(match.group(1)) if match else 0 diff --git a/dendrite/browser/sync_api/_ext_impl/browserbase/_impl.py b/dendrite/browser/sync_api/_ext_impl/browserbase/_impl.py deleted file mode 100644 index 138a6e4..0000000 --- a/dendrite/browser/sync_api/_ext_impl/browserbase/_impl.py +++ /dev/null @@ -1,60 +0,0 @@ -from typing import TYPE_CHECKING, Optional -from dendrite.browser._common._exceptions.dendrite_exception import BrowserNotLaunchedError -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.remote.browserbase_config import BrowserbaseConfig - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser.sync_api._ext_impl.browserbase._client import BrowserbaseClient -from playwright.sync_api import Playwright -from loguru import logger -from dendrite.browser.sync_api._ext_impl.browserbase._download import BrowserbaseDownload - - -class BrowserBaseImpl(ImplBrowser): - - def __init__(self, settings: BrowserbaseConfig) -> None: - self.settings = settings - self._client = BrowserbaseClient( - self.settings.api_key, self.settings.project_id - ) - self._session_id: Optional[str] = None - - def stop_session(self): - if self._session_id: - self._client.stop_session(self._session_id) - - def start_browser(self, playwright: Playwright, pw_options: dict): - logger.debug("Starting browser") - self._session_id = self._client.create_session() - url = self._client.connect_url(self.settings.enable_proxy, self._session_id) - logger.debug(f"Connecting to browser at {url}") - return playwright.chromium.connect_over_cdp(url) - - def configure_context(self, browser: "Dendrite"): - logger.debug("Configuring browser context") - page = browser.get_active_page() - pw_page = page.playwright_page - if browser.browser_context is None: - raise BrowserNotLaunchedError() - client = browser.browser_context.new_cdp_session(pw_page) - client.send( - "Browser.setDownloadBehavior", - {"behavior": "allow", "downloadPath": "downloads", "eventsEnabled": True}, - ) - - def get_download( - self, - dendrite_browser: "Dendrite", - pw_page: PlaywrightPage, - timeout: float = 30000, - ) -> BrowserbaseDownload: - if not self._session_id: - raise ValueError( - "Downloads are not enabled for this provider. Specify enable_downloads=True in the constructor" - ) - logger.debug("Getting download") - download = dendrite_browser._download_handler.get_data(pw_page, timeout) - self._client.save_downloads_on_disk(self._session_id, download.path(), 30) - return BrowserbaseDownload(self._session_id, download, self._client) diff --git a/dendrite/browser/sync_api/_ext_impl/browserless/__init__.py b/dendrite/browser/sync_api/_ext_impl/browserless/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_ext_impl/browserless/_impl.py b/dendrite/browser/sync_api/_ext_impl/browserless/_impl.py deleted file mode 100644 index 2e2646f..0000000 --- a/dendrite/browser/sync_api/_ext_impl/browserless/_impl.py +++ /dev/null @@ -1,53 +0,0 @@ -import json -from typing import TYPE_CHECKING, Optional -from dendrite.browser._common._exceptions.dendrite_exception import BrowserNotLaunchedError -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.remote.browserless_config import BrowserlessConfig - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser.sync_api._ext_impl.browserbase._client import BrowserbaseClient -from playwright.sync_api import Playwright -from loguru import logger -import urllib.parse -from dendrite.browser.sync_api._ext_impl.browserbase._download import BrowserbaseDownload - - -class BrowserlessImpl(ImplBrowser): - - def __init__(self, settings: BrowserlessConfig) -> None: - self.settings = settings - self._session_id: Optional[str] = None - - def stop_session(self): - pass - - def start_browser(self, playwright: Playwright, pw_options: dict): - logger.debug("Starting browser") - url = self._format_connection_url(pw_options) - logger.debug(f"Connecting to browser at {url}") - return playwright.chromium.connect_over_cdp(url) - - def _format_connection_url(self, pw_options: dict) -> str: - url = self.settings.url.rstrip("?").rstrip("/") - query = { - "token": self.settings.api_key, - "blockAds": self.settings.block_ads, - "launch": json.dumps(pw_options), - } - if self.settings.proxy: - query["proxy"] = (self.settings.proxy,) - query["proxyCountry"] = (self.settings.proxy_country,) - return f"{url}?{urllib.parse.urlencode(query)}" - - def configure_context(self, browser: "Dendrite"): - pass - - def get_download( - self, - dendrite_browser: "Dendrite", - pw_page: PlaywrightPage, - timeout: float = 30000, - ) -> BrowserbaseDownload: - raise NotImplementedError("Downloads are not supported for Browserless") diff --git a/dendrite/exceptions/__init__.py b/dendrite/exceptions/__init__.py index 7aff356..ad0fbf7 100644 --- a/dendrite/exceptions/__init__.py +++ b/dendrite/exceptions/__init__.py @@ -1,11 +1,11 @@ from ..browser._common._exceptions.dendrite_exception import ( BaseDendriteException, + BrowserNotLaunchedError, DendriteException, IncorrectOutcomeError, InvalidAuthSessionError, MissingApiKeyError, PageConditionNotMet, - BrowserNotLaunchedError, ) __all__ = [ diff --git a/dendrite/logic/local/ask/ask.py b/dendrite/logic/ask/ask.py similarity index 84% rename from dendrite/logic/local/ask/ask.py rename to dendrite/logic/ask/ask.py index 7464826..272ee1b 100644 --- a/dendrite/logic/local/ask/ask.py +++ b/dendrite/logic/ask/ask.py @@ -1,14 +1,16 @@ -import json import re -from anthropic.types import TextBlock -from jsonschema import validate +from typing import List + import json_repair +from jsonschema import validate -from .image import segment_image +from dendrite.logic.llm.agent import Agent, Message +from dendrite.logic.llm.config import llm_config +from dendrite.models.dto.ask_page_dto import AskPageDTO +from dendrite.models.response.ask_page_response import AskPageResponse -from dendrite_server_merge.dto.AskPageDTO import AskPageDTO -from dendrite_server_merge.responses.AskPageResponse import AskPageResponse +from .image import segment_image async def ask_page_action( @@ -18,31 +20,24 @@ async def ask_page_action( ask_page_dto.page_information.screenshot_base64, segment_height=2000 ) + agent = Agent(llm_config.get("ask_page_agent")) scrolled_to_segment_i = 0 content = generate_ask_page_prompt(ask_page_dto, image_segments) - messages = [{"role": "user", "content": content},] + messages: List[Message] = [ + {"role": "user", "content": content}, + ] max_iterations = len(image_segments) + 5 iteration = 0 while iteration < max_iterations: iteration += 1 - config = { - "messages": messages, - "model": "claude-3-5-sonnet-20241022", - "temperature": 0.3, - "max_tokens": 1500, - } - res = await async_claude_request(config, ask_page_dto.api_config) - - if not isinstance(res.content[0], TextBlock): - raise Exception("Needs to be an text block") - - text = res.content[0].text - dict_res = { + + + text = await agent.call_llm(messages) + messages.append({ "role": "assistant", "content": text, - } - messages.append(dict_res) + }) json_pattern = r"```json(.*?)```" @@ -59,7 +54,7 @@ async def ask_page_action( if not isinstance(data_dict, dict): content = "Your message doesn't contain a correctly formatted json object, try again." - messages.append({"role": "user","content": content}) + messages.append({"role": "user", "content": content}) continue if "scroll_down" in data_dict: @@ -70,14 +65,12 @@ async def ask_page_action( content = "You cannot scroll any further." messages.append({"role": "user", "content": content}) continue - + elif "return_data" in data_dict and "description" in data_dict: return_data = data_dict["return_data"] try: if ask_page_dto.return_schema: - validate( - instance=return_data, schema=ask_page_dto.return_schema - ) + validate(instance=return_data, schema=ask_page_dto.return_schema) except Exception as e: err_message = "Your return data doesn't match the requested return json schema, try again. Exception: {e}" messages.append( @@ -103,7 +96,9 @@ async def ask_page_action( ) else: - err_message = "Your message doesn't contain a correctly formatted action, try again." + err_message = ( + "Your message doesn't contain a correctly formatted action, try again." + ) messages.append( { "role": "user", @@ -118,10 +113,9 @@ async def ask_page_action( ) - - - -def generate_ask_page_prompt(ask_page_dto: AskPageDTO, image_segments: list, scrolled_to_segment_i: int = 0) -> list: +def generate_ask_page_prompt( + ask_page_dto: AskPageDTO, image_segments: list, scrolled_to_segment_i: int = 0 +) -> list: # Generate scroll down hint based on number of segments scroll_down_hint = ( "" @@ -202,14 +196,15 @@ def generate_ask_page_prompt(ask_page_dto: AskPageDTO, image_segments: list, scr return content + def generate_scroll_prompt(image_segments: list, next_segment: int) -> list: """ Generates the prompt for scrolling to next segment. - + Args: image_segments: List of image segments next_segment: Index of next segment - + Returns: List of message content blocks """ @@ -218,7 +213,7 @@ def generate_scroll_prompt(image_segments: list, next_segment: int) -> list: if next_segment == len(image_segments) - 1 else "" ) - + content = [ { "type": "text", @@ -233,5 +228,5 @@ def generate_scroll_prompt(image_segments: list, next_segment: int) -> list: }, }, ] - + return content diff --git a/dendrite/logic/local/ask/image.py b/dendrite/logic/ask/image.py similarity index 100% rename from dendrite/logic/local/ask/image.py rename to dendrite/logic/ask/image.py index a8681ff..6a61566 100644 --- a/dendrite/logic/local/ask/image.py +++ b/dendrite/logic/ask/image.py @@ -1,9 +1,9 @@ -import io import base64 +import io from typing import List -from PIL import Image from loguru import logger +from PIL import Image def segment_image( diff --git a/dendrite/logic/cache/element_cache.py b/dendrite/logic/cache/element_cache.py index e69de29..650ea59 100644 --- a/dendrite/logic/cache/element_cache.py +++ b/dendrite/logic/cache/element_cache.py @@ -0,0 +1,8 @@ +from typing import Generic, TypeVar, TypedDict +from pydantic import BaseModel + +from dendrite.logic.cache.file_cache import FileCache +from dendrite.models.scripts import Script +from dendrite.models.selector import Selector + +element_cache = FileCache(Selector, "./cache/get_element.json") diff --git a/dendrite/logic/cache/extract_cache.py b/dendrite/logic/cache/extract_cache.py new file mode 100644 index 0000000..2d5d8f3 --- /dev/null +++ b/dendrite/logic/cache/extract_cache.py @@ -0,0 +1,5 @@ +from dendrite.models.scripts import Script + +from .file_cache import FileCache + +ExtractCache = FileCache(Script, "./cache/extract.json") diff --git a/dendrite/logic/cache/file_cache.py b/dendrite/logic/cache/file_cache.py index f4bcf61..8b9afd7 100644 --- a/dendrite/logic/cache/file_cache.py +++ b/dendrite/logic/cache/file_cache.py @@ -1,14 +1,16 @@ -from pathlib import Path import json import threading -from typing import Generic, TypeVar, Union, Type, Dict -from pydantic import BaseModel from hashlib import md5 +from pathlib import Path +from typing import Dict, Generic, Type, TypeVar, Union + +from pydantic import BaseModel + +T = TypeVar("T", bound=BaseModel) -T = TypeVar('T', bound=BaseModel) class FileCache(Generic[T]): - def __init__(self, model_class: Type[T], filepath: str = './cache.json'): + def __init__(self, model_class: Type[T], filepath: Union[str, Path] = "./cache.json"): self.filepath = Path(filepath) self.model_class = model_class self.lock = threading.RLock() @@ -27,7 +29,7 @@ def _load_cache(self) -> None: try: json_string = self.filepath.read_text() raw_dict = json.loads(json_string) - + # Convert each entry back to the model class self.cache = { k: self.model_class.model_validate_json(json.dumps(v)) @@ -41,16 +43,15 @@ def _save_cache(self, cache_dict: Dict[str, T]) -> None: with self.lock: # Convert models to dict before saving serializable_dict = { - k: json.loads(v.model_dump_json()) - for k, v in cache_dict.items() + k: json.loads(v.model_dump_json()) for k, v in cache_dict.items() } self.filepath.write_text(json.dumps(serializable_dict, indent=2)) - def get(self, key: str) -> Union[T, None]: + def get(self, key: Union[str,Dict[str,str]]) -> Union[T, None]: hashed_key = self.hash(key) return self.cache.get(hashed_key) - def set(self, key: str, value: T) -> None: + def set(self, key: Union[str, Dict[str, str]], value: T) -> None: hashed_key = self.hash(key) with self.lock: self.cache[hashed_key] = value @@ -63,5 +64,36 @@ def delete(self, key: str) -> None: del self.cache[hashed_key] self._save_cache(self.cache) - def hash(self, key: str) -> str: - return md5(key.encode()).hexdigest() + def hash(self, key: Union[str, Dict]) -> str: + """ + Create a deterministic hash from a string or dictionary. + Handles nested structures and different value types. + """ + + def normalize_value(v): + if isinstance(v, dict): + return self.hash(v) + elif isinstance(v, (list, tuple)): + return "[" + ",".join(normalize_value(x) for x in v) + "]" + elif v is None: + return "null" + elif isinstance(v, bool): + return str(v).lower() + else: + return str(v).strip() + + if isinstance(key, dict): + try: + # Sort by normalized string keys + sorted_pairs = [ + f"{str(k).strip()}∴{normalize_value(v)}" # Using a rare Unicode character as delimiter + for k, v in sorted(key.items(), key=lambda x: str(x[0]).strip()) + ] + key = "❘".join(sorted_pairs) # Using another rare Unicode character + except Exception as e: + raise ValueError(f"Failed to process dictionary key: {e}") + + try: + return md5(str(key).encode("utf-8")).hexdigest() + except Exception as e: + raise ValueError(f"Failed to create hash: {e}") diff --git a/dendrite/logic/cache/utils.py b/dendrite/logic/cache/utils.py new file mode 100644 index 0000000..dc78418 --- /dev/null +++ b/dendrite/logic/cache/utils.py @@ -0,0 +1,19 @@ + + +from datetime import datetime +from typing import Optional +from urllib.parse import urlparse + +from dendrite.logic.cache import extract_cache +from dendrite.models.scripts import Script + + +def save_script(code: str, prompt: str, url: str): + domain = urlparse(url).netloc + script = Script( + url=url, domain=domain, script=code, created_at=datetime.now().isoformat() + ) + extract_cache.ExtractCache.set({"prompt": prompt, "domain": domain}, script) + +def get_script(prompt: str, domain: str) -> Optional[Script]: + return extract_cache.ExtractCache.get({"prompt": prompt, "domain": domain}) \ No newline at end of file diff --git a/dendrite/logic/local/code/code_session.py b/dendrite/logic/code/code_session.py similarity index 87% rename from dendrite/logic/local/code/code_session.py rename to dendrite/logic/code/code_session.py index e6458c0..6d93d11 100644 --- a/dendrite/logic/local/code/code_session.py +++ b/dendrite/logic/code/code_session.py @@ -1,14 +1,15 @@ +import json # Important to keep since it is used inside the scripts +import re # Important to keep since it is used inside the scripts import sys import traceback -import re # Important to keep since it is used inside the scripts -import json # Important to keep since it is used inside the scripts from datetime import datetime # Important to keep since it is used inside the scripts - from typing import Any, List, Optional + from bs4 import BeautifulSoup +from jsonschema import validate from loguru import logger + from ..dom.truncate import truncate_long_string -from jsonschema import validate class InterpreterError(Exception): @@ -135,3 +136,25 @@ def llm_readable_exec_res( response += f"\n\nDo these variables match the expected values? Remember, this is what the user asked for:\n\n{prompt}\n\nIf not, try again and remember, if one approach fails several times you might need to reinspect the DOM and try a different approach. You have {max_attempts - attempts} attempts left to try and complete the task. If you are happy with the results, output a success message." return response + + +def execute(script: str, raw_html: str, return_data_json_schema) -> Any: + code_session = CodeSession() + soup = BeautifulSoup(raw_html, "lxml") + try: + + created_variables = code_session.exec_code(script, soup, raw_html) + + if "response_data" in created_variables: + response_data = created_variables["response_data"] + + try: + code_session.validate_response(return_data_json_schema, response_data) + except Exception as e: + raise Exception(f"Failed to validate response data. Exception: {e}") + + return response_data + else: + raise Exception("No return data available for this script.") + except Exception as e: + raise e diff --git a/dendrite/logic/config.py b/dendrite/logic/config.py new file mode 100644 index 0000000..028da35 --- /dev/null +++ b/dendrite/logic/config.py @@ -0,0 +1,15 @@ +from pathlib import Path +from dendrite.logic.cache.file_cache import FileCache +from dendrite.models.scripts import Script +from dendrite.models.selector import Selector + + +class Config: + def __init__(self): + self.cache_path = Path("./cache") + self.llm_config = "8udjsad" + self.extract_cache = FileCache(Script, self.cache_path / "extract.json") + self.element_cache = FileCache(Selector, self.cache_path / "get_element.json") + + +config = Config() \ No newline at end of file diff --git a/dendrite/logic/local/dom/css.py b/dendrite/logic/dom/css.py similarity index 98% rename from dendrite/logic/local/dom/css.py rename to dendrite/logic/dom/css.py index a64833e..7ae582c 100644 --- a/dendrite/logic/local/dom/css.py +++ b/dendrite/logic/dom/css.py @@ -1,9 +1,9 @@ from typing import Optional + from bs4 import BeautifulSoup, Tag from loguru import logger - def find_css_selector(ele: Tag, soup: BeautifulSoup) -> str: if ele is None: return "" @@ -103,6 +103,7 @@ def position_in_node_list(element: Tag, parent: Tag): return index + 1 return -1 + # https://github.com/mathiasbynens/CSS.escape def css_escape(value): if len(str(value)) == 0: @@ -151,8 +152,11 @@ def css_escape(value): return result + def check_if_selector_successful( - selector: str, bs4: BeautifulSoup, only_one: bool, + selector: str, + bs4: BeautifulSoup, + only_one: bool, ) -> Optional[str]: els = None @@ -167,4 +171,4 @@ def check_if_selector_successful( elif not only_one and len(els) >= 1: return selector - return None \ No newline at end of file + return None diff --git a/dendrite/logic/dom/strip.py b/dendrite/logic/dom/strip.py new file mode 100644 index 0000000..e10cfbc --- /dev/null +++ b/dendrite/logic/dom/strip.py @@ -0,0 +1,159 @@ +import copy +from typing import List, Union, overload + +from bs4 import BeautifulSoup, Comment, Doctype, Tag + + +def mild_strip(soup: Tag, keep_d_id: bool = True) -> BeautifulSoup: + new_soup = BeautifulSoup(str(soup), "html.parser") + _mild_strip(new_soup, keep_d_id) + return new_soup + + +def mild_strip_in_place(soup: BeautifulSoup, keep_d_id: bool = True) -> None: + _mild_strip(soup, keep_d_id) + + +def _mild_strip(soup: BeautifulSoup, keep_d_id: bool = True) -> None: + for element in soup(text=lambda text: isinstance(text, Comment)): + element.extract() + + # for text in soup.find_all(text=lambda text: isinstance(text, NavigableString)): + # if len(text) > 200: + # text.replace_with(text[:200] + f"... [{len(text)-200} more chars]") + + for tag in soup( + ["head", "script", "style", "path", "polygon", "defs", "svg", "br", "Doctype"] + ): + tag.extract() + + for element in soup.contents: + if isinstance(element, Doctype): + element.extract() + + # for tag in soup.find_all(True): + # tag.attrs = { + # attr: (value[:100] if isinstance(value, str) else value) + # for attr, value in tag.attrs.items() + # } + # if keep_d_id == False: + # del tag["d-id"] + for tag in soup.find_all(True): + if tag.attrs.get("is-interactable-d_id") == "true": + continue + + tag.attrs = { + attr: (value[:100] if isinstance(value, str) else value) + for attr, value in tag.attrs.items() + } + if keep_d_id == False: + del tag["d-id"] + + # if browser != None: + # for elem in list(soup.descendants): + # if isinstance(elem, Tag) and not browser.element_is_visible(elem): + # elem.extract() + + +@overload +def shorten_attr_val(value: str, limit: int = 50) -> str: ... + + +@overload +def shorten_attr_val(value: List[str], limit: int = 50) -> List[str]: ... + + +def shorten_attr_val( + value: Union[str, List[str]], limit: int = 50 +) -> Union[str, List[str]]: + if isinstance(value, str): + return value[:limit] + + char_count = sum(map(len, value)) + if char_count <= limit: + return value + + while len(value) > 1 and char_count > limit: + char_count -= len(value.pop()) + + if len(value) == 1: + return value[0][:limit] + + return value + + +def clear_attrs(element: Tag): + + salient_attributes = [ + "d-id", + "class", + "id", + "type", + "alt", + "aria-describedby", + "aria-label", + "contenteditable", + "aria-role", + "input-checked", + "label", + "name", + "option_selected", + "placeholder", + "readonly", + "text-value", + "title", + "value", + "href", + "role", + "action", + "method", + ] + attrs = { + attr: shorten_attr_val(value, limit=200) + for attr, value in element.attrs.items() + if attr in salient_attributes + } + element.attrs = attrs + + +def strip_soup(soup: BeautifulSoup) -> BeautifulSoup: + # Create a copy of the soup to avoid modifying the original + stripped_soup = BeautifulSoup(str(soup), "html.parser") + + for tag in stripped_soup( + [ + "head", + "script", + "style", + "path", + "polygon", + "defs", + "br", + "Doctype", + ] # add noscript? + ): + tag.extract() + + # Remove comments + comments = stripped_soup.find_all(text=lambda text: isinstance(text, Comment)) + for comment in comments: + comment.extract() + + # Clear non-salient attributes + for element in stripped_soup.find_all(True): + if isinstance(element, Doctype): + element.extract() + else: + clear_attrs(element) + + return stripped_soup + + + +def remove_hidden_elements(soup: BeautifulSoup): + # data-hidden is added by DendriteBrowser when an element is not visible + new_soup = copy.copy(soup) + elems = new_soup.find_all(attrs={"data-hidden": True}) + for elem in elems: + elem.extract() + return new_soup \ No newline at end of file diff --git a/dendrite/logic/local/dom/truncate.py b/dendrite/logic/dom/truncate.py similarity index 99% rename from dendrite/logic/local/dom/truncate.py rename to dendrite/logic/dom/truncate.py index 6e09fc2..fa1bfd8 100644 --- a/dendrite/logic/local/dom/truncate.py +++ b/dendrite/logic/dom/truncate.py @@ -1,5 +1,6 @@ import re + def truncate_long_string( val: str, max_len_start: int = 150, diff --git a/dendrite/logic/extract/cached_script.py b/dendrite/logic/extract/cached_script.py new file mode 100644 index 0000000..7554d4d --- /dev/null +++ b/dendrite/logic/extract/cached_script.py @@ -0,0 +1,42 @@ +from typing import Any, List, Optional, Tuple +from urllib.parse import urlparse + +from loguru import logger + +from dendrite.logic.cache.utils import get_script +from dendrite.logic.code.code_session import execute +from dendrite.models.scripts import Script + + +async def get_working_cached_script( + prompt: str, + raw_html: str, + url: str, + return_data_json_schema: Any, +) -> Optional[Tuple[Script, Any]]: + domain = urlparse(url).netloc + + if len(url) == 0: + raise Exception("Domain must be specified") + + scripts: List[Script] = [get_script(prompt, domain) or ...] + logger.debug( + f"Found {len(scripts)} scripts in cache | Prompt: {prompt} in domain: {domain}" + ) + + for script in scripts: + try: + res = execute(script.script, raw_html, return_data_json_schema) + return script, res + except Exception as e: + logger.debug( + f"Script failed with error: {str(e)} | Prompt: {prompt} in domain: {domain}" + ) + continue + + if len(scripts) == 0: + return None + + raise Exception( + f"No working script found in cache even though {len(scripts)} scripts were available | Prompt: '{prompt}' in domain: '{domain}'" + ) \ No newline at end of file diff --git a/dendrite/logic/extract/compress_html.py b/dendrite/logic/extract/compress_html.py new file mode 100644 index 0000000..af09f5b --- /dev/null +++ b/dendrite/logic/extract/compress_html.py @@ -0,0 +1,491 @@ +import re +import time +from collections import Counter +from typing import List, Optional, Tuple, TypedDict, Union + +from bs4 import BeautifulSoup, NavigableString, PageElement +from bs4.element import Tag + +from dendrite.logic.dom.truncate import ( + truncate_and_remove_whitespace, + truncate_long_string_w_words, +) +from dendrite.logic.llm.token_count import token_count + +MAX_REPEATING_ELEMENT_AMOUNT = 6 + + +class FollowableListInfo(TypedDict): + expanded_elements: List[Tag] + amount: int + parent_element_d_id: str + first_element_d_id: str + + +class CompressHTML: + def __init__( + self, + root_soup: Union[BeautifulSoup, Tag], + ids_to_expand: List[str] = [], + compression_multiplier: float = 1, + exclude_dendrite_ids=False, + max_token_size: int = 80000, + max_size_per_element: int = 6000, + focus_on_text=False, + ) -> None: + if exclude_dendrite_ids == True: + for tag in root_soup.find_all(): + if "d-id" in tag.attrs: + del tag["d-id"] + + self.orginal_size = len(str(root_soup)) + self.root = BeautifulSoup(str(root_soup), "html.parser") + self.original_root = BeautifulSoup(str(root_soup), "html.parser") + self.ids_to_expand = ids_to_expand + self.expand_crawlable_list = False + self.compression_multiplier = compression_multiplier + self.lists_with_followable_urls: List[FollowableListInfo] = [] + self.max_token_size = max_token_size + self.max_size_per_element = max_size_per_element + self.focus_on_text = focus_on_text + self.search_terms = [] + + def get_lists_with_followable_urls(self): + return self.lists_with_followable_urls + + def _remove_consecutive_newlines(self, text: str, max_newlines=1): + cleaned_text = re.sub(r"\n{2,}", "\n" * max_newlines, text) + return cleaned_text + + def _parent_is_explicitly_expanded(self, tag: Tag) -> bool: + for tag in tag.parents: + if tag.get("d-id", None) in self.ids_to_expand: + return True + return False + + def _should_expand_anyways(self, tag: Tag) -> bool: + curr_id = tag.get("d-id", None) + + if curr_id in self.ids_to_expand: + return True + + tag_descendants = [ + descendant for descendant in tag.descendants if isinstance(descendant, Tag) + ] + for tag in tag_descendants: + id = tag.get("d-id", None) + if id in self.ids_to_expand: + return True + + for parent in tag.parents: + id = parent.get("d-id", None) + if id in self.ids_to_expand: + return True + # Expand the children of expanded elements if the expanded element isn't too big + if len(str(parent)) > 4000: + return False + + return False + + def clear_attrs(self, element: Tag, unique_class_names: List[str]): + attrs = {} + class_attr = element.get("class", []) + salient_attributes = [ + "type" "alt", + "aria-describedby", + "aria-label", + "aria-role", + "input-checked", + "label", + "name", + "option_selected", + "placeholder", + "readonly", + "text-value", + "title", + "value", + "href", + ] + + attrs = { + attr: (str(value)[:100] if len(str(value)) > 100 else str(value)) + for attr, value in element.attrs.items() + if attr in salient_attributes + } + + if class_attr: + if isinstance(class_attr, str): + class_attr = class_attr.split(" ") + + class_name_len = 0 + class_max_len = 200 + classes_to_show = [] + for class_name in class_attr: + if class_name_len + len(class_name) < class_max_len: + classes_to_show.append(class_name) + class_name_len += len(class_name) + + if len(classes_to_show) > 0: + attrs = {**attrs, "class": " ".join(classes_to_show)} + + id = element.get("id") + d_id = element.get("d-id") + + if isinstance(id, str): + attrs = {**attrs, "id": id} + + if d_id: + attrs = {**attrs, "d-id": d_id} + + element.attrs = attrs + + def extract_crawlable_list( + self, repeating_element_sequence_ids: List[str], amount_repeating_left: int + ): + items: List[Tag] = [] + parent_element_d_id: str = "" + first_element_d_id = repeating_element_sequence_ids[0] + + for d_id in repeating_element_sequence_ids: + + el = self.original_root.find(attrs={"d-id": str(d_id)}) + if ( + parent_element_d_id == "" + and isinstance(el, Tag) + and isinstance(el.parent, Tag) + ): + parent_element_d_id = str(el.parent.get("d-id", "")) + + original = BeautifulSoup(str(el), "html.parser") + link = original.find("a") + if link and isinstance(original, Tag): + items.append(original) + + if ( + len(items) == len(repeating_element_sequence_ids) + and len(items) >= MAX_REPEATING_ELEMENT_AMOUNT + and parent_element_d_id != "" + ): + self.lists_with_followable_urls.append( + { + "amount": len(items) + amount_repeating_left, + "expanded_elements": items, + "parent_element_d_id": parent_element_d_id, + "first_element_d_id": first_element_d_id, + } + ) + + def get_html_display(self) -> str: + def collapse(element: PageElement) -> str: + chars_to_keep = 2000 if self.focus_on_text else 100 + + if isinstance(element, Tag): + if element.get("d-id", "") == "-1": + return "" + + text = element.get_text() + if text: + element.attrs["is-compressed"] = "true" + element.attrs["d-id"] = str(element.get("d-id", "")) + element.clear() + element.append( + truncate_and_remove_whitespace( + text, max_len_start=chars_to_keep, max_len_end=chars_to_keep + ) + ) + return str(element) + else: + return "" + elif isinstance(element, NavigableString): + return truncate_and_remove_whitespace( + element, max_len_start=chars_to_keep, max_len_end=chars_to_keep + ) + else: + return "" + + start_time = time.time() + class_names = [ + name for tag in self.root.find_all() for name in tag.get("class", []) + ] + + counts = Counter(class_names) + unique_class_names = [name for name, count in counts.items() if count == 1] + + def get_repeating_element_info(el: Tag) -> Tuple[str, List[str]]: + return ( + el.name, + [el.name for el in el.children if isinstance(el, Tag)], + ) + + def is_repeating_element( + previous_element_info: Optional[Tuple[str, List[str]]], element: Tag + ) -> bool: + if previous_element_info: + repeat_element_info = get_repeating_element_info(element) + return ( + previous_element_info == repeat_element_info + and element.name != "div" + ) + + return False + + # children_size += token_count(str(child)) + # if children_size > 400: + # children_left = {} + # for c in child.next_siblings: + # if isinstance(c, Tag): + # if c.name in children_left: + # children_left[c.name] += 1 + # else: + # children_left[c.name] = 0 + # desc = "" + # for c_name in children_left.keys(): + # desc = f"{children_left[c_name]} {c_name} tag(s) truncated for readability" + # child.replace_with(f"[...{desc}...]") + # break + + def traverse(tag: Union[BeautifulSoup, Tag]): + previous_element_info: Optional[Tuple[str, List[str]]] = None + repeating_element_sequence_ids = [] + has_placed_truncation = False + same_element_repeat_amount: int = 0 + + tag_children = (child for child in tag.children if isinstance(child, Tag)) + + total_token_size = 0 + for index, child in enumerate(tag_children): + + total_token_size += len(str(child)) + # if total_token_size > self.max_size_per_element * 4 and index > 60: + # names = {} + # for next_sibling in child.next_siblings: + # if isinstance(next_sibling, Tag): + # if next_sibling.name in names: + # names[next_sibling.name] += 1 + # else: + # names[next_sibling.name] = 1 + + # removable = [sib for sib in child.next_siblings] + # for sib in removable: + # try: + # sib.replace_with("") + # except: + # print("failed to replace sib: ", str(sib)) + + # truncation_message = [] + # for element_name, amount_hidden in names.items(): + # truncation_message.append( + # f"{amount_hidden} `{element_name}` element(s)" + # ) + + # child.replace_with( + # f"[...{','.join(truncation_message)} hidden for readablity ...]" + # ) + # break + + repeating_element_sequence_ids.append(child.get("d-id", "None")) + + if is_repeating_element(previous_element_info, child): + same_element_repeat_amount += 1 + + if ( + same_element_repeat_amount > MAX_REPEATING_ELEMENT_AMOUNT + and self._parent_is_explicitly_expanded(child) == False + ): + amount_repeating = 0 + if isinstance(child, Tag): + for sibling in child.next_siblings: + if isinstance(sibling, Tag) and is_repeating_element( + previous_element_info, sibling + ): + amount_repeating += 1 + + if has_placed_truncation == False and amount_repeating >= 1: + child.replace_with( + f"[...{amount_repeating} repeating `{child.name}` elements collapsed for readability...]" + ) + has_placed_truncation = True + + self.extract_crawlable_list( + repeating_element_sequence_ids, amount_repeating + ) + + if self.expand_crawlable_list == True: + for d_id in repeating_element_sequence_ids: + sequence_element = self.root.find( + attrs={"d-id": str(d_id)} + ) + + if isinstance(sequence_element, Tag): + original = BeautifulSoup( + str( + self.original_root.find( + attrs={"d-id": str(d_id)} + ) + ), + "html.parser", + ) + links = original.find_all("a") + for link in links: + + self.ids_to_expand.append( + str(link.get("d-id", "None")) + ) + sequence_element.replace_with(original) + traverse(sequence_element) + + repeating_element_sequence_ids = [] + else: + child.replace_with("") + continue + + else: + has_placed_truncation = False + previous_element_info = get_repeating_element_info(child) + same_element_repeat_amount = 0 + + # If a parent is expanded, allow larger element until collapsing + compression_mod = self.compression_multiplier + if self._parent_is_explicitly_expanded(child): + compression_mod = 0.5 + + if len(str(child)) < self.orginal_size // 300 * compression_mod: + if self._should_expand_anyways(child): + traverse(child) + else: + chars_to_keep = 2000 if self.focus_on_text else 80 + truncated_text = truncate_long_string_w_words( + child.get_text().replace("\n", ""), + max_len_start=chars_to_keep, + max_len_end=chars_to_keep, + ) + if truncated_text.strip(): + child.attrs = { + "is-compressed": "true", + "d-id": str(child.get("d-id", "")), + } + child.string = truncated_text + else: + child.replace_with("") + elif len(str(child)) > self.orginal_size // 10 * compression_mod: + traverse(child) + else: + if self._should_expand_anyways(child): + traverse(child) + else: + replacement = collapse(child) + child.replace_with(BeautifulSoup(replacement, "html.parser")) + + # total_token_size += len(str(child)) + # print("total_token_size: ", total_token_size) + + # if total_token_size > 2000: + # next_element_tags = [ + # sibling.name for sibling in child.next_siblings if isinstance(sibling, Tag)] + # child.replace_with( + # f"[...{', '.join(next_element_tags)} tags collapsed for readability...]") + + def remove_double_nested(soup): + for tag in soup.find_all(True): + # If a tag only contains a single child of the same type + if len(tag.find_all(True, recursive=False)) == 1 and isinstance( + tag.contents[0], Tag + ): + child_tag = tag.contents[0] + # move the contents of the child tag up to the parent + tag.clear() + tag.extend(child_tag.contents) + if len(tag.find_all(True, recursive=False)) == 1 and isinstance( + tag.contents[0], Tag + ): + remove_double_nested(tag) + + return soup + + def is_effectively_empty(element): + if element.name and not element.attrs: + if not element.contents or all( + isinstance(child, NavigableString) and len(child.strip()) < 3 + for child in element.contents + ): + return True + return False + + start_time = time.time() + for i in range(10): + for element in self.root.find_all(is_effectively_empty): + element.decompose() + + for tag in self.root.find_all(): + self.clear_attrs(tag, unique_class_names) + + if len(str(self.root)) < 1500: + return self.root.prettify() + + + # print("time: ", end_time - start_time) + + # remove_double_nested(self.root) + # clean_attributes(root, keep_dendrite_id=False) + traverse(self.root) + # print("traverse time: ", end_time - start_time) + + return self.root.prettify() + + def get_compression_level(self) -> Tuple[str, int]: + if self.orginal_size > 100000: + return "4/4 (Extremely compressed)", 4 + elif self.orginal_size > 40000: + return "3/4 (Very compressed)", 3 + elif self.orginal_size > 4000: + return "2/4 (Slightly compressed)", 2 + elif self.orginal_size > 400: + return "1/4 (Very mild compression)", 1 + else: + return "0/4 (no compression)", 0 + + async def compress(self, search_terms: List[str] = []) -> str: + iterations = 0 + pretty = "" + self.search_terms = search_terms + + while token_count(pretty) > self.max_token_size or pretty == "": + iterations += 1 + if iterations > 5: + break + compression_level_desc, _ = self.get_compression_level() + # Show elements with relevant search terms more + if len(self.search_terms) > 0: + + def contains_text(element): + if element: + # Check only direct text content, not including nested elements + direct_text = "".join( + child + for child in element.children + if isinstance(child, NavigableString) + ).lower() + return any( + term.lower() in direct_text for term in self.search_terms + ) + return False + + matching_elements = self.original_root.find_all(contains_text) + for element in matching_elements: + print(f"Element contains search word: {str(element)[:400]}") + d_id = element.get("d-id") + if d_id: + self.ids_to_expand.append(d_id) + + # print("old: ", self.orginal_size) + md = self.get_html_display() + md = self._remove_consecutive_newlines(md) + pretty = BeautifulSoup(md, "html.parser").prettify() + end = time.time() + # print("pretty: ", pretty) + # print("new: ", token_count(pretty)) + # print("took: ", end - start) + # print("compression_level: ", compression_level_desc) + self.compression_multiplier *= 2 + + return pretty diff --git a/dendrite/logic/extract/extract.py b/dendrite/logic/extract/extract.py new file mode 100644 index 0000000..957d92f --- /dev/null +++ b/dendrite/logic/extract/extract.py @@ -0,0 +1,165 @@ +import asyncio +import hashlib +from typing import Optional +from urllib.parse import urlparse + +from loguru import logger + +from dendrite.logic.extract.cached_script import get_working_cached_script +from dendrite.logic.extract.extract_agent import ExtractAgent +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.response.extract_response import ExtractResponse + +# Assuming you have these imports +# from your_module import WebScrapingAgent, run_script_if_cached + + +async def test_cache( + extract_dto: ExtractDTO +) -> Optional[ExtractResponse]: + try: + + cached_script_res = await get_working_cached_script( + extract_dto.combined_prompt, + extract_dto.page_information.raw_html, + extract_dto.page_information.url, + extract_dto.return_data_json_schema + ) + + if cached_script_res is None: + return None + + script, script_exec_res = cached_script_res + return ExtractResponse( + status="success", + message="Re-used a preexisting script from cache with the same specifications.", + return_data=script_exec_res, + used_cache=True, + created_script=script.script, + ) + + except Exception as e: + return ExtractResponse( + status="failed", + message=str(e), + ) + + +class InMemoryLockManager: + # Class-level dictionaries to keep track of locks and events + locks = {} + events = {} + global_lock = asyncio.Lock() + + def __init__(self, extract_page_dto: ExtractDTO,): + self.key = self.generate_key(extract_page_dto) + + def generate_key(self, extract_page_dto: ExtractDTO) -> str: + domain = urlparse(extract_page_dto.page_information.url).netloc + key_data = f"{domain}:{extract_page_dto.combined_prompt}" + return hashlib.sha256(key_data.encode()).hexdigest() + + async def acquire_lock(self, timeout: int = 60) -> bool: + async with InMemoryLockManager.global_lock: + if self.key in InMemoryLockManager.locks: + # Lock is already acquired + return False + else: + # Acquire the lock + InMemoryLockManager.locks[self.key] = True + return True + + async def release_lock(self): + async with InMemoryLockManager.global_lock: + InMemoryLockManager.locks.pop(self.key, None) + InMemoryLockManager.events.pop(self.key, None) + + async def publish(self, message: str): + async with InMemoryLockManager.global_lock: + event = InMemoryLockManager.events.get(self.key) + if event: + event.set() + + async def subscribe(self): + async with InMemoryLockManager.global_lock: + if self.key not in InMemoryLockManager.events: + InMemoryLockManager.events[self.key] = asyncio.Event() + # No need to assign to self.event; return the event instead + return InMemoryLockManager.events[self.key] + + async def wait_for_notification( + self, event: asyncio.Event, timeout: float = 1600.0 + ) -> bool: + try: + await asyncio.wait_for(event.wait(), timeout) + return True + except asyncio.TimeoutError as e: + logger.error(f"Timeout error: {e}") + return False + finally: + # Clean up event + async with InMemoryLockManager.global_lock: + InMemoryLockManager.events.pop(self.key, None) + + + +async def extract( + extract_page_dto: ExtractDTO +) -> ExtractResponse: + # Check cache usage flags + if extract_page_dto.use_cache or extract_page_dto.force_use_cache: + res = await test_cache(extract_page_dto) + if res: + return res + + if extract_page_dto.force_use_cache: + return ExtractResponse( + status="failed", + message="No script available in cache that matches this prompt.", + ) + + # Proceed with lock acquisition and processing + lock_manager = InMemoryLockManager(extract_page_dto) + lock_acquired = await lock_manager.acquire_lock() + + if lock_acquired: + return await generate_script(extract_page_dto, lock_manager) + else: + res = await wait_for_script_generation(extract_page_dto, lock_manager) + + if res: + return res + # Else create a working script since page is different + extract_agent = ExtractAgent( + extract_page_dto.page_information, + ) + res = await extract_agent.write_and_run_script(extract_page_dto) + return res + + + +async def generate_script(extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager) -> ExtractResponse: + try: + extract_agent = ExtractAgent( + extract_page_dto.page_information, + + ) + res = await extract_agent.write_and_run_script(extract_page_dto) + await lock_manager.publish("done") + return res + except Exception as e: + await lock_manager.publish("failed") + raise e + finally: + await lock_manager.release_lock() + +async def wait_for_script_generation(extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager) -> Optional[ExtractResponse]: + event = await lock_manager.subscribe() + logger.info("Waiting for script to be generated") + notification_received = await lock_manager.wait_for_notification(event) + + # If script was created after waiting + if notification_received: + res = await test_cache(extract_page_dto) + if res: + return res diff --git a/dendrite/logic/extract/extract_agent.py b/dendrite/logic/extract/extract_agent.py new file mode 100644 index 0000000..44e9660 --- /dev/null +++ b/dendrite/logic/extract/extract_agent.py @@ -0,0 +1,281 @@ +import json +import re +from typing import Any, List, Optional + +from dendrite.logic.cache.utils import save_script +from dendrite.logic.dom.strip import mild_strip +from dendrite.logic.extract.prompts import ( + LARGE_HTML_CHAR_TRUNCATE_LEN, + create_script_prompt_segmented_html, +) + +from dendrite.logic.extract.scroll_agent import ScrollAgent +from dendrite.logic.llm.agent import Agent +from dendrite.logic.get_element.hanifi_search import get_expanded_dom +from dendrite.logic.llm.token_count import token_count +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.page_information import PageInformation +from dendrite.logic.llm.agent import Message + +from bs4 import BeautifulSoup + +from dendrite.models.response.extract_response import ExtractResponse +from ..code.code_session import CodeSession +from ..ask.image import segment_image +from ..llm.config import llm_config + + + +class ExtractAgent(Agent): + def __init__( + self, + page_information: PageInformation, + ) -> None: + super().__init__(llm_config.get("extract_agent")) + self.page_information = page_information + self.soup = BeautifulSoup(page_information.raw_html, "lxml") + self.messages = [] + self.generated_script: Optional[str] = None + self.llm_config = llm_config + + + def get_generated_script(self): + return self.generated_script + + async def write_and_run_script( + self, extract_page_dto: ExtractDTO + ) -> ExtractResponse: + mild_soup = mild_strip(self.soup) + + search_terms = [] + + segments = segment_image( + extract_page_dto.page_information.screenshot_base64, segment_height=4000 + ) + + scroll_agent = ScrollAgent( + self.llm_config.get("scroll_agent"), self.page_information + ) + scroll_result = await scroll_agent.scroll_through_page( + extract_page_dto.combined_prompt, + image_segments=segments, + ) + + if scroll_result.status == "error": + return ExtractResponse( + status="impossible", + message=str(scroll_result.message), + ) + + if scroll_result.status == "loading": + return ExtractResponse( + status="loading", + message="This page is still loading. Please wait a bit longer.", + ) + + expanded_html = None + + if scroll_result.element_to_inspect_html: + combined_prompt = ( + "Get these elements (make sure you only return element that you are confident that these are the correct elements, it's OK to not select any elements):\n- " + + "\n- ".join(scroll_result.element_to_inspect_html) + ) + expanded = await get_expanded_dom( + mild_soup, + combined_prompt, + ) + if expanded: + expanded_html = expanded[0] + + if expanded_html: + return await self.code_script_from_found_expanded_html_tags( + extract_page_dto, expanded_html, segments + ) + + raise Exception("Failed to extract data from the page") # TODO: skriv bättre + + def segment_large_tag(self, tag): + segments = [] + current_segment = "" + current_tokens = 0 + for line in tag.split("\n"): + line_tokens = token_count(line) + if current_tokens + line_tokens > 4000: + segments.append(current_segment) + current_segment = line + current_tokens = line_tokens + else: + current_segment += line + "\n" + current_tokens += line_tokens + if current_segment: + segments.append(current_segment) + return segments + + async def code_script_from_found_expanded_html_tags( + self, extract_page_dto: ExtractDTO, expanded_html, segments + ): + # agent_logger.info("Starting code_script_from_found_expanded_html_tags method") + messages = [] + + user_prompt = create_script_prompt_segmented_html( + extract_page_dto.combined_prompt, + expanded_html, + self.page_information.url, + ) + # agent_logger.debug(f"User prompt created: {user_prompt[:100]}...") + + content = { + "type": "text", + "text": user_prompt, + } + + messages: List[Message] = [ + {"role": "user", "content": user_prompt}, + ] + + iterations = 0 + max_retries = 10 + + generated_script: str = "" + response_data: Any | None = None + + while iterations <= max_retries: + iterations += 1 + # agent_logger.info(f"Starting iteration {iterations}") + + text = await self.call_llm(messages) + messages.append({"role": "assistant", "content": text}) + + json_pattern = r"```json(.*?)```" + code_pattern = r"```python(.*?)```" + + if text: + json_matches = re.findall(json_pattern, text, re.DOTALL) + code_matches = re.findall(code_pattern, text, re.DOTALL) + + if len(json_matches) + len(code_matches) > 1: + content = "Error: Please output only one action at a time (either JSON or Python code, not both)." + messages.append({"role": "user", "content": content}) + continue + + for code_match in code_matches: + # agent_logger.debug("Processing code match") + generated_script = code_match.strip() + temp_code_session = CodeSession() + try: + variables = temp_code_session.exec_code( + generated_script, + self.soup, + self.page_information.raw_html, + ) + # agent_logger.debug("Code execution successful") + except Exception as e: + # agent_logger.error(f"Code execution failed: {str(e)}") + content = f"Error: {str(e)}" + messages.append({"role": "user", "content": content}) + continue + + try: + if "response_data" in variables: + response_data = variables["response_data"] + # agent_logger.debug(f"Response data: {response_data}") + + if extract_page_dto.return_data_json_schema != None: + temp_code_session.validate_response( + extract_page_dto.return_data_json_schema, + response_data, + ) + + llm_readable_exec_res = ( + temp_code_session.llm_readable_exec_res( + variables, + extract_page_dto.combined_prompt, + iterations, + max_retries, + ) + ) + + messages.append( + {"role": "user", "content": llm_readable_exec_res} + ) + continue + else: + content = ( + f"Error: You need to add the variable 'response_data'" + ) + messages.append( + { + "role": "user", + "content": content, + } + ) + continue + except Exception as e: + llm_readable_exec_res = temp_code_session.llm_readable_exec_res( + variables, + extract_page_dto.combined_prompt, + iterations, + max_retries, + ) + content = f"Error: Failed to validate `response_data`. Exception: {e}. {llm_readable_exec_res}" + messages.append( + { + "role": "user", + "content": content, + } + ) + continue + + for json_match in json_matches: + # agent_logger.debug("Processing JSON match") + extracted_json = json_match.strip() + data_dict = json.loads(extracted_json) + current_segment = 0 + if "request_more_html" in data_dict: + # agent_logger.info("Processing element indexes") + try: + current_segment += 1 + content = f"""Here is more of the HTML:\n```html\n{expanded_html[LARGE_HTML_CHAR_TRUNCATE_LEN*current_segment:LARGE_HTML_CHAR_TRUNCATE_LEN*(current_segment+1)]}\n```""" + if len(expanded_html) > LARGE_HTML_CHAR_TRUNCATE_LEN * ( + current_segment + 1 + ): + content += "\nThere is still more HTML to see. You can request more if needed." + else: + content += "\nThis is the end of the HTML content." + messages.append({"role": "user", "content": content}) + continue + except Exception as e: + # agent_logger.error( + # f"Error processing element indexes: {str(e)}" + # ) + content = f"Error: {str(e)}" + messages.append({"role": "user", "content": content}) + continue + elif "error" in data_dict: + # agent_logger.error(f"Error in data_dict: {data_dict['error']}") + raise Exception(data_dict["error"]) + elif "success" in data_dict: + # agent_logger.info("Script generation successful") + + self.generated_script = generated_script + save_script( + self.generated_script, + extract_page_dto.combined_prompt, + self.page_information.url, + ) + + # agent_logger.debug(f"Response data: {response_data}") + return ExtractResponse( + status="success", + message=data_dict["success"], + return_data=response_data, + created_script=self.get_generated_script(), + ) + + # agent_logger.warning("Failed to create script after retrying several times") + return ExtractResponse( + status="failed", + message="Failed to create script after retrying several times.", + return_data=None, + created_script=self.get_generated_script(), + ) diff --git a/dendrite/logic/local/extract/prompts.py b/dendrite/logic/extract/prompts.py similarity index 99% rename from dendrite/logic/local/extract/prompts.py rename to dendrite/logic/extract/prompts.py index 4171ce7..35c1037 100644 --- a/dendrite/logic/local/extract/prompts.py +++ b/dendrite/logic/extract/prompts.py @@ -76,7 +76,7 @@ def expand_futher_prompt( END EXAMPLE OUTPUT""" -def create_script_prompt_compressed_html( +def generate_prompt_extract_compressed_html( combined_prompt: str, expanded_html: str, current_url: str, @@ -160,7 +160,7 @@ def create_script_prompt_segmented_html( expanded_html: str, current_url: str, ): - if len(expanded_html)/4 > LARGE_HTML_CHAR_TRUNCATE_LEN: + if len(expanded_html) / 4 > LARGE_HTML_CHAR_TRUNCATE_LEN: html_prompt = f"""```html {expanded_html[:LARGE_HTML_CHAR_TRUNCATE_LEN]} ``` diff --git a/dendrite/logic/local/extract/scroll_agent.py b/dendrite/logic/extract/scroll_agent.py similarity index 89% rename from dendrite/logic/local/extract/scroll_agent.py rename to dendrite/logic/extract/scroll_agent.py index aa40ed8..32869ba 100644 --- a/dendrite/logic/local/extract/scroll_agent.py +++ b/dendrite/logic/extract/scroll_agent.py @@ -1,15 +1,13 @@ -from abc import ABC, abstractmethod -from dataclasses import dataclass import json import re +from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import List, Literal, Optional -from anthropic.types import TextBlock from loguru import logger +from dendrite.logic.llm.agent import Agent, Message from dendrite.models.page_information import PageInformation -from dendrite_server_merge.core.llm.claude import async_claude_request - ScrollActionStatus = Literal["done", "scroll_down", "loading", "error"] @@ -61,9 +59,9 @@ def parse(self, data_dict: dict, segment_i: int) -> Optional[ScrollResult]: return None -class ScrollAgent: - def __init__(self, api_config, page_information: PageInformation): - self.api_config = api_config +class ScrollAgent(Agent): + def __init__(self, llm_config, page_information: PageInformation): + super().__init__(llm_config.get("scroll_agent")) self.page_information = page_information self.choices: List[ScrollRes] = [ ElementPromptsAction(), @@ -110,19 +108,9 @@ async def scroll_through_page( return ScrollResult(all_elements_to_inspect_html, current_segment, "done") - async def process_segment(self, messages: List[dict]) -> dict: - config = { - "messages": messages, - "model": "claude-3-5-sonnet-20241022", - "temperature": 0.3, - "max_tokens": 1500, - } - res = await async_claude_request(config, self.api_config) - - if not isinstance(res.content[0], TextBlock): - raise Exception("Response needs to be a text block") + async def process_segment(self, messages: List[Message]) -> dict: - text = res.content[0].text + text = await self.call_llm(messages) messages.append({"role": "assistant", "content": text}) json_pattern = r"```json(.*?)```" @@ -133,6 +121,7 @@ async def process_segment(self, messages: List[dict]) -> dict: logger.warning("Agent output multiple actions in one message") error_message = "Error: Please output only one action at a time." messages.append({"role": "user", "content": error_message}) + raise Exception(error_message) elif json_matches: return json.loads(json_matches[0].strip()) @@ -140,10 +129,11 @@ async def process_segment(self, messages: List[dict]) -> dict: logger.error(error_message) messages.append({"role": "user", "content": error_message}) raise Exception(error_message) + def create_initial_message( self, combined_prompt: str, first_image: str - ) -> List[dict]: + ) -> List[Message]: content = [ { "type": "text", @@ -222,17 +212,15 @@ def create_initial_message( {"role": "user", "content": content}, ] - def create_scroll_message(self, image: str) -> dict: + def create_scroll_message(self, image: str) -> Message: return { "role": "user", "content": [ {"type": "text", "text": "Scrolled down, here is the new viewport:"}, { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": image, + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image}", }, }, ], diff --git a/dendrite/logic/factory.py b/dendrite/logic/factory.py index 542ae4f..c3416bf 100644 --- a/dendrite/logic/factory.py +++ b/dendrite/logic/factory.py @@ -1,17 +1,16 @@ -from typing import Literal, Optional +# from typing import Literal, Optional -from dendrite.logic.interfaces.async_api import BrowserAPIProtocol +# from dendrite.logic.interfaces.async_api import LogicAPIProtocol -class BrowserAPIFactory: - @staticmethod - def create_browser_api( - mode: Literal["local", "remote"], - api_config: APIConfig, - session_id: Optional[str] = None - ) -> BrowserAPIProtocol: - if mode == "local": - return LocalBrowserAPI() - else: - return BrowserAPIClient(api_config, session_id) \ No newline at end of file +# class BrowserAPIFactory: +# @staticmethod +# def create_browser_api( +# mode: Literal["local", "remote"], +# session_id: Optional[str] = None +# ) -> LogicAPIProtocol': +# if mode == "local": +# return LocalBrowserAPI() +# else: +# return BrowserAPIClient(api_config, session_id) \ No newline at end of file diff --git a/dendrite/logic/get_element/agents/prompts/__init__.py b/dendrite/logic/get_element/agents/prompts/__init__.py new file mode 100644 index 0000000..acf88e2 --- /dev/null +++ b/dendrite/logic/get_element/agents/prompts/__init__.py @@ -0,0 +1,12 @@ +def load_prompt(prompt_path: str) -> str: + with open(prompt_path, "r") as f: + prompt = f.read() + return prompt + + +SEGMENT_PROMPT = load_prompt( + "dendrite/logic/get_element/agents/prompts/segment.prompt" +) +SELECT_PROMPT = load_prompt( + "dendrite/logic/get_element/agents/prompts/segment.prompt" +) diff --git a/dendrite/logic/local/get_element/agents/prompts/segment.prompt b/dendrite/logic/get_element/agents/prompts/segment.prompt similarity index 100% rename from dendrite/logic/local/get_element/agents/prompts/segment.prompt rename to dendrite/logic/get_element/agents/prompts/segment.prompt diff --git a/dendrite/logic/local/get_element/agents/prompts/select.prompt b/dendrite/logic/get_element/agents/prompts/select.prompt similarity index 100% rename from dendrite/logic/local/get_element/agents/prompts/select.prompt rename to dendrite/logic/get_element/agents/prompts/select.prompt diff --git a/dendrite/logic/local/get_element/agents/segment_agent.py b/dendrite/logic/get_element/agents/segment_agent.py similarity index 70% rename from dendrite/logic/local/get_element/agents/segment_agent.py rename to dendrite/logic/get_element/agents/segment_agent.py index 1251582..8ebadc0 100644 --- a/dendrite/logic/local/get_element/agents/segment_agent.py +++ b/dendrite/logic/get_element/agents/segment_agent.py @@ -1,21 +1,15 @@ -import re import json -from typing import Annotated, List, Literal, Tuple, Union +import re +from typing import Annotated, List, Literal, Union + from annotated_types import Len from loguru import logger from pydantic import BaseModel, ValidationError -from anthropic.types import Message, TextBlock - -from dendrite.browser.async_api._core.models.api_config import APIConfig - -from .agent import ( - AnthropicAgent, -) -from .prompts import ( - SEGMENT_PROMPT, -) +from dendrite.logic.llm.agent import Agent +from dendrite.logic.llm.config import llm_config +from .prompts import SEGMENT_PROMPT class SegmentAgentSuccessResponse(BaseModel): @@ -36,16 +30,9 @@ class SegmentAgentFailureResponse(BaseModel): ] -def parse_claude_result(result: Message, index: int) -> SegmentAgentReponseType: +def parse_segment_output(text: str, index: int) -> SegmentAgentReponseType: json_pattern = r"```json(.*?)```" - model = None - - if len(result.content) == 0 or not isinstance(result.content[0], TextBlock): - return SegmentAgentFailureResponse( - reason="No content from agent", status="failed", index=index - ) - - text = result.content[0].text + res = None if text is None: return SegmentAgentFailureResponse( @@ -72,7 +59,7 @@ def parse_claude_result(result: Message, index: int) -> SegmentAgentReponseType: reason="No d_ids provided", status="failed", index=index ) - model = SegmentAgentSuccessResponse( + res = SegmentAgentSuccessResponse( reason=json_data["reason"], status="success", d_id=json_data["d_id"], @@ -80,31 +67,27 @@ def parse_claude_result(result: Message, index: int) -> SegmentAgentReponseType: except json.JSONDecodeError as e: raise ValueError(f"Failed to decode JSON: {e}") - if model is None: + if res is None: try: - model = SegmentAgentFailureResponse.model_validate_json(json_matches[0]) + res = SegmentAgentFailureResponse.model_validate_json(json_matches[0]) except ValidationError as e: logger.bind(json=json_matches[0]).error( f"Failed to parse JSON: {e}", ) - model = SegmentAgentFailureResponse( + res = SegmentAgentFailureResponse( reason="Failed to parse JSON", status="failed", index=index ) - model.index = index - return model - + res.index = index + return res async def extract_relevant_d_ids( prompt: str, segments: List[str], - api_config: APIConfig, index: int, -) -> Tuple[int, int, SegmentAgentReponseType]: - agent = AnthropicAgent( - "claude-3-haiku-20240307", api_config, system_message=SEGMENT_PROMPT - ) +) -> SegmentAgentReponseType: + agent = Agent(llm_config.get("segment_agent"), system_prompt=SEGMENT_PROMPT) message = "" for segment in segments: message += ( @@ -122,23 +105,17 @@ async def extract_relevant_d_ids( continue try: - parsed_res = parse_claude_result(res, index) + parsed_res = parse_segment_output(res, index) # If we successfully parsed the result, return it - completion = res.usage.output_tokens if res.usage else 0 - prompt_token = res.usage.input_tokens if res.usage else 0 - return (prompt_token, completion, parsed_res) + return parsed_res except Exception as e: # If we encounter a ValueError, ask the agent to correct its output logger.warning(f"Error in segment agent: {e}") message = f"An exception occurred in your output: {e}\n\nPlease correct your output and try again. Ensure you're providing a valid JSON response." # If we've exhausted all retries, return a failure response - return ( - 0, - 0, - SegmentAgentFailureResponse( - reason="Max retries reached without successful parsing", - status="failed", - index=index, - ), + return SegmentAgentFailureResponse( + reason="Max retries reached without successful parsing", + status="failed", + index=index, ) diff --git a/dendrite/logic/local/get_element/agents/select_agent.py b/dendrite/logic/get_element/agents/select_agent.py similarity index 79% rename from dendrite/logic/local/get_element/agents/select_agent.py rename to dendrite/logic/get_element/agents/select_agent.py index f377960..658274c 100644 --- a/dendrite/logic/local/get_element/agents/select_agent.py +++ b/dendrite/logic/get_element/agents/select_agent.py @@ -1,24 +1,21 @@ import re from typing import List, Optional, Tuple -from pydantic import BaseModel -from anthropic.types import Message, TextBlock -from dendrite.browser.async_api._core.models.api_config import APIConfig + +from loguru import logger from openai.types.chat import ChatCompletion +from pydantic import BaseModel from dendrite.browser._common.types import Status -from .agent import ( - AnthropicAgent, -) -from .prompts import ( - SELECT_PROMPT, -) -from ..dom import SelectedTag +from dendrite.logic.llm.agent import Agent +from dendrite.logic.llm.config import llm_config +from ..hanifi_segment import SelectedTag +from .prompts import SELECT_PROMPT class SelectAgentResponse(BaseModel): reason: str - d_ids: Optional[List[str]] = None + d_id: Optional[List[str]] = None status: Status @@ -26,14 +23,11 @@ async def select_best_tag( expanded_html_tree: str, tags: List[SelectedTag], prompt: str, - api_config: APIConfig, time_since_frame_navigated: Optional[float], return_several: bool = False, ) -> Tuple[int, int, Optional[SelectAgentResponse]]: - agent = AnthropicAgent( - "claude-3-5-sonnet-20241022", api_config, system_message=SELECT_PROMPT - ) + agent = Agent(llm_config.get("select_agent"), system_prompt=SELECT_PROMPT) message = f"\n{prompt}\n" @@ -50,23 +44,18 @@ async def select_best_tag( message += f"""\n\nThis page was first loaded {round(time_since_frame_navigated, 2)} second(s) ago. If the page is blank or the data is not available on the current page it could be because the page is still loading.\n\nDon't return an element that isn't what the user asked for, in this case it is better to return `status: impossible` or `status: loading` if you think the page is still loading.""" res = await agent.add_message(message) - # messages = agent.dump_messages() - # with open("select_agent_messages.json", "w") as f: - # f.write(messages) - parsed = await parse_select_response(res) + logger.info(f"Select agent response: {res}") + + parsed = await parse_select_output(res) # token_usage = res.usage.input_tokens + res.usage.output_tokens return (0, 0, parsed) -async def parse_select_response(result: Message) -> Optional[SelectAgentResponse]: +async def parse_select_output(text: str) -> Optional[SelectAgentResponse]: json_pattern = r"```json(.*?)```" - if not isinstance(result.content[0], TextBlock): - return None - - text = result.content[0].text json_matches = re.findall(json_pattern, text, re.DOTALL) if not json_matches: diff --git a/dendrite/logic/get_element/cached_selector.py b/dendrite/logic/get_element/cached_selector.py new file mode 100644 index 0000000..9f8e782 --- /dev/null +++ b/dendrite/logic/get_element/cached_selector.py @@ -0,0 +1,35 @@ +from datetime import datetime +from typing import Optional, Type +from urllib.parse import urlparse + +from bs4 import BeautifulSoup +from loguru import logger +from pydantic import BaseModel + + +from dendrite.logic.cache.file_cache import FileCache +from dendrite.models.selector import Selector +from dendrite.logic.config import config + + +async def get_selector_from_cache( + url: str, prompt: str, cache: FileCache[Selector] +) -> Optional[Selector]: + netloc = urlparse(url).netloc + + return cache.get({"netloc": netloc, "prompt": prompt}) + + +async def add_selector_to_cache(prompt: str, bs4_selector: str, url: str) -> None: + cache = config.element_cache + created_at = datetime.now().isoformat() + netloc = urlparse(url).netloc + selector: Selector = Selector( + prompt=prompt, + selector=bs4_selector, + url=url, + netloc=netloc, + created_at=created_at, + ) + + cache.set({"netloc": netloc, "prompt": prompt}, selector) \ No newline at end of file diff --git a/dendrite/logic/get_element/get_element.py b/dendrite/logic/get_element/get_element.py new file mode 100644 index 0000000..af694d9 --- /dev/null +++ b/dendrite/logic/get_element/get_element.py @@ -0,0 +1,112 @@ +from typing import Optional + +from bs4 import BeautifulSoup, Tag +from loguru import logger +from pydantic import BaseModel + +from dendrite.logic.cache.file_cache import FileCache +from dendrite.logic.config import config +from dendrite.logic.dom.css import check_if_selector_successful, find_css_selector +from dendrite.logic.dom.strip import remove_hidden_elements +from dendrite.logic.get_element.cached_selector import ( + add_selector_to_cache, + get_selector_from_cache, +) +from dendrite.models.dto.get_elements_dto import GetElementsDTO +from dendrite.models.page_information import PageInformation +from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.selector import Selector + +from .hanifi_search import hanifi_search + + + + +async def get_element(dto: GetElementsDTO) -> GetElementResponse: + + if isinstance(dto.prompt, str): + return await process_prompt(dto.prompt, dto) + raise ... + +async def process_prompt( + prompt: str, dto: GetElementsDTO +) -> GetElementResponse: + + soup = BeautifulSoup(dto.page_information.raw_html, "lxml") + + if dto.use_cache: + res = await check_cache( + soup, + dto.page_information.url, + prompt, + dto.only_one, + ) + if res: + return res + + if dto.force_use_cache: + return GetElementResponse( + selectors=[], + status="failed", + message="Forced to use cache, but no cached selectors found", + used_cache=False, + ) + + return await get_new_element(soup, prompt, dto ) + +async def get_new_element(soup: BeautifulSoup, prompt: str, dto: GetElementsDTO) -> GetElementResponse: + soup_without_hidden_elements = remove_hidden_elements(soup) + element = await hanifi_search( + soup_without_hidden_elements, + prompt, + dto.page_information.time_since_frame_navigated, + ) + interactable = element[0] + + if interactable.status == "success": + if interactable.dendrite_id is None: + interactable.status = "failed" + interactable.reason = "No d-id found returned from agent" + tag = soup.find(attrs={"d-id": interactable.dendrite_id}) + if isinstance(tag, Tag): + selector = find_css_selector(tag, soup) + await add_selector_to_cache( + prompt, + bs4_selector=selector, + url=dto.page_information.url, + ) + return GetElementResponse( + selectors=[selector], + message=interactable.reason, + d_id=interactable.dendrite_id, + status="success", + used_cache=False, + ) + interactable.status = "failed" + interactable.reason = "d-id does not exist in the soup" + + return GetElementResponse( + message=interactable.reason, + status=interactable.status, + used_cache=False, + ) + + +async def check_cache( + soup: BeautifulSoup, url: str, prompt: str, only_one: bool +) -> Optional[GetElementResponse]: + cache = config.element_cache + db_selectors = await get_selector_from_cache(url, prompt, cache) + + if db_selectors is None: + return None + + successful_selectors = [] + + if check_if_selector_successful(db_selectors.selector, soup, only_one): + return GetElementResponse( + selectors=successful_selectors, + status="success", + used_cache=True, + ) + diff --git a/dendrite/logic/local/get_element/hanifi_search.py b/dendrite/logic/get_element/hanifi_search.py similarity index 69% rename from dendrite/logic/local/get_element/hanifi_search.py rename to dendrite/logic/get_element/hanifi_search.py index b10c715..03449d5 100644 --- a/dendrite/logic/local/get_element/hanifi_search.py +++ b/dendrite/logic/get_element/hanifi_search.py @@ -1,36 +1,28 @@ import asyncio from typing import Any, Coroutine, List, Optional, Tuple, Union -from bs4 import BeautifulSoup, Tag - -from dendrite.browser.async_api._core.models.api_config import APIConfig +from bs4 import BeautifulSoup, Tag -from .agents import select_agent -from .agents import segment_agent +from dendrite.logic.dom.strip import strip_soup +from dendrite.logic.llm.config import llm_config +from .agents import segment_agent, select_agent from .agents.segment_agent import ( SegmentAgentFailureResponse, SegmentAgentReponseType, SegmentAgentSuccessResponse, - -) -from .dom import ( - SelectedTag, - expand_tags, - hanifi_segment, - strip_soup, -) -from .models import ( - Interactable, + extract_relevant_d_ids, ) +from .hanifi_segment import SelectedTag, expand_tags, hanifi_segment +from .models import Element async def get_expanded_dom( - soup: BeautifulSoup, prompt: str, api_config: APIConfig + soup: BeautifulSoup, prompt: str ) -> Optional[Tuple[str, List[SegmentAgentReponseType], List[SelectedTag]]]: new_nodes = hanifi_segment(soup, 6000, 3) - tags = await get_relevant_tags(prompt, new_nodes, api_config, soup) + tags = await get_relevant_tags(prompt, new_nodes) succesful_d_ids = [ (tag.d_id, tag.index, tag.reason) @@ -56,17 +48,16 @@ async def get_expanded_dom( async def hanifi_search( soup: BeautifulSoup, prompt: str, - api_config: APIConfig, time_since_frame_navigated: Optional[float] = None, return_several: bool = False, -) -> List[Interactable]: - +) -> List[Element]: + stripped_soup = strip_soup(soup) - expand_res = await get_expanded_dom(stripped_soup, prompt, api_config) + expand_res = await get_expanded_dom(stripped_soup, prompt) if expand_res is None: return [ - Interactable(status="failed", reason="No element found when expanding HTML") + Element(status="failed", reason="No element found when expanding HTML") ] expanded, tags, flat_list = expand_res @@ -80,64 +71,52 @@ async def hanifi_search( succesful_tags.append(tag) if len(succesful_tags) == 0: - return [Interactable(status="failed", reason="No relevant tags found in DOM")] + return [Element(status="failed", reason="No relevant tags found in DOM")] (input_token, output_token, res) = await select_agent.select_best_tag( expanded, flat_list, prompt, - api_config, time_since_frame_navigated, return_several, ) if not res: - return [Interactable(status="failed", reason="Failed to get interactable")] + return [Element(status="failed", reason="Failed to get element")] - if res.d_ids: + if res.d_id: if return_several: return [ - Interactable(status=res.status, dendrite_id=d_id, reason=res.reason) - for d_id in res.d_ids + Element(status=res.status, dendrite_id=d_id, reason=res.reason) + for d_id in res.d_id ] else: return [ - Interactable( - status=res.status, dendrite_id=res.d_ids[0], reason=res.reason + Element( + status=res.status, dendrite_id=res.d_id[0], reason=res.reason ) ] - return [Interactable(status=res.status, dendrite_id=None, reason=res.reason)] + return [Element(status=res.status, dendrite_id=None, reason=res.reason)] async def get_relevant_tags( prompt: str, segments: List[List[str]], - api_config: APIConfig, - soup: BeautifulSoup, ) -> List[SegmentAgentReponseType]: - tasks: List[Coroutine[Any, Any, Tuple[int, int, SegmentAgentReponseType]]] = [] + + tasks: List[Coroutine[Any, Any, SegmentAgentReponseType]] = [] + for index, segment in enumerate(segments): tasks.append( - segment_agent.extract_relevant_d_ids(prompt, segment, api_config, index) + extract_relevant_d_ids(prompt, segment, index) ) - results: List[Tuple[int, int, SegmentAgentReponseType]] = await asyncio.gather( - *tasks - ) + results: List[SegmentAgentReponseType] = await asyncio.gather(*tasks) if results is None: - return - - tokens_prompt = 0 - tokens_completion = 0 - - tags = [] - for res in results: - tokens_prompt += res[0] - tokens_completion += res[1] - tags.append(res[2]) + return [] - return tags + return results def get_if_one_tag( diff --git a/dendrite/logic/local/get_element/dom.py b/dendrite/logic/get_element/hanifi_segment.py similarity index 75% rename from dendrite/logic/local/get_element/dom.py rename to dendrite/logic/get_element/hanifi_segment.py index 071b6e7..5c6a092 100644 --- a/dendrite/logic/local/get_element/dom.py +++ b/dendrite/logic/get_element/hanifi_segment.py @@ -1,116 +1,11 @@ import copy -from dataclasses import dataclass from collections import deque -from typing import List, Optional, Union, overload -from bs4 import BeautifulSoup, Comment, Doctype, NavigableString, Tag -from ..dom.truncate import truncate_and_remove_whitespace, truncate_long_string_w_words - - - -@overload -def shorten_attr_val(value: str, limit: int = 50) -> str: ... - - -@overload -def shorten_attr_val(value: List[str], limit: int = 50) -> List[str]: ... - - -def shorten_attr_val( - value: Union[str, List[str]], limit: int = 50 -) -> Union[str, List[str]]: - if isinstance(value, str): - return value[:limit] - - char_count = sum(map(len, value)) - if char_count <= limit: - return value - - while len(value) > 1 and char_count > limit: - char_count -= len(value.pop()) - - if len(value) == 1: - return value[0][:limit] - - return value - - -def clear_attrs(element: Tag): - - salient_attributes = [ - "d-id", - "class", - "id", - "type", - "alt", - "aria-describedby", - "aria-label", - "contenteditable", - "aria-role", - "input-checked", - "label", - "name", - "option_selected", - "placeholder", - "readonly", - "text-value", - "title", - "value", - "href", - "role", - "action", - "method", - ] - attrs = { - attr: shorten_attr_val(value, limit=200) - for attr, value in element.attrs.items() - if attr in salient_attributes - } - element.attrs = attrs - - -def strip_soup(soup: BeautifulSoup) -> BeautifulSoup: - # Create a copy of the soup to avoid modifying the original - stripped_soup = BeautifulSoup(str(soup), "html.parser") - - for tag in stripped_soup( - [ - "head", - "script", - "style", - "path", - "polygon", - "defs", - "br", - "Doctype", - ] # add noscript? - ): - tag.extract() - - # Remove comments - comments = stripped_soup.find_all(text=lambda text: isinstance(text, Comment)) - for comment in comments: - comment.extract() - - # Clear non-salient attributes - for element in stripped_soup.find_all(True): - if isinstance(element, Doctype): - element.extract() - else: - clear_attrs(element) - - return stripped_soup - -import copy - +from dataclasses import dataclass +from typing import List, Optional, Union +from bs4 import BeautifulSoup, Comment, Doctype, NavigableString, Tag -def remove_hidden_elements(soup: BeautifulSoup): - # data-hidden is added by DendriteBrowser when an element is not visible - new_soup = copy.copy(soup) - elems = new_soup.find_all(attrs={"data-hidden": True}) - for elem in elems: - elem.extract() - return new_soup +from ..dom.truncate import truncate_and_remove_whitespace, truncate_long_string_w_words # Define a threshold (e.g., 30% of the total document size) diff --git a/dendrite/logic/local/get_element/models.py b/dendrite/logic/get_element/models.py similarity index 93% rename from dendrite/logic/local/get_element/models.py rename to dendrite/logic/get_element/models.py index ead7476..c13f842 100644 --- a/dendrite/logic/local/get_element/models.py +++ b/dendrite/logic/get_element/models.py @@ -10,7 +10,7 @@ class ExpandedTag(NamedTuple): @dataclass -class Interactable: +class Element: status: Status reason: str dendrite_id: Optional[str] = None diff --git a/dendrite/logic/hosted/_api/_http_client.py b/dendrite/logic/hosted/_api/_http_client.py index 72777e2..55f01fc 100644 --- a/dendrite/logic/hosted/_api/_http_client.py +++ b/dendrite/logic/hosted/_api/_http_client.py @@ -1,14 +1,17 @@ import os -from typing import Optional +from typing import Any, Dict, Optional + import httpx from loguru import logger -from dendrite.browser.async_api._core.models.api_config import APIConfig +from dendrite.models.api_config import APIConfig + class HTTPClient: def __init__(self, api_config: APIConfig, session_id: Optional[str] = None): self.api_key = api_config.dendrite_api_key + self.api_config = api_config self.session_id = session_id self.base_url = self.resolve_base_url() @@ -24,7 +27,7 @@ async def send_request( self, endpoint: str, params: Optional[dict] = None, - data: Optional[dict] = None, + data: Optional[Dict[str,Any]] = None, headers: Optional[dict] = None, method: str = "GET", ) -> httpx.Response: @@ -39,6 +42,12 @@ async def send_request( async with httpx.AsyncClient(timeout=300) as client: try: + + # inject api_config to data + if data: + data["api_config"] = self.api_config.model_dump() + + response = await client.request( method, url, params=params, json=data, headers=headers ) diff --git a/dendrite/logic/hosted/_api/browser_api_client.py b/dendrite/logic/hosted/_api/browser_api_client.py index b52738f..0b79090 100644 --- a/dendrite/logic/hosted/_api/browser_api_client.py +++ b/dendrite/logic/hosted/_api/browser_api_client.py @@ -1,84 +1,37 @@ -from typing import Optional - -from loguru import logger -from dendrite.browser.async_api._api.response.cache_extract_response import ( - CacheExtractResponse, -) -from dendrite.browser.async_api._api.response.selector_cache_response import ( - SelectorCacheResponse, -) from dendrite.browser.async_api._core.models.authentication import AuthSession -from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse -from dendrite.browser.async_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.browser.async_api._api.dto.get_elements_dto import GetElementsDTO -from dendrite.browser.async_api._api.dto.make_interaction_dto import MakeInteractionDTO -from dendrite.browser.async_api._api.dto.extract_dto import ExtractDTO -from dendrite.browser.async_api._api.dto.try_run_script_dto import TryRunScriptDTO -from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.browser.async_api._api.response.ask_page_response import AskPageResponse -from dendrite.browser.async_api._api.response.interaction_response import ( - InteractionResponse, -) -from dendrite.browser.async_api._api.response.extract_response import ExtractResponse -from dendrite.browser.async_api._api._http_client import HTTPClient -from dendrite.browser._common._exceptions.dendrite_exception import ( - InvalidAuthSessionError, -) -from dendrite.browser.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO +from dendrite.logic.hosted._api._http_client import HTTPClient +from dendrite.models.dto.ask_page_dto import AskPageDTO +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.dto.get_elements_dto import GetElementsDTO +from dendrite.models.dto.make_interaction_dto import VerifyActionDTO +from dendrite.models.response.ask_page_response import AskPageResponse +from dendrite.models.response.extract_response import ExtractResponse +from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.response.interaction_response import InteractionResponse class BrowserAPIClient(HTTPClient): - async def authenticate(self, dto: AuthenticateDTO): - res = await self.send_request( - "actions/authenticate", data=dto.model_dump(), method="POST" - ) - - if res.status_code == 204: - raise InvalidAuthSessionError(domain=dto.domains) - - return AuthSession(**res.json()) - - async def upload_auth_session(self, dto: UploadAuthSessionDTO): - await self.send_request( - "actions/upload-auth-session", data=dto.dict(), method="POST" - ) - - async def check_selector_cache( - self, dto: CheckSelectorCacheDTO - ) -> SelectorCacheResponse: - res = await self.send_request( - "actions/check-selector-cache", data=dto.dict(), method="POST" - ) - return SelectorCacheResponse(**res.json()) - - async def get_interactions_selector( + async def get_element( self, dto: GetElementsDTO ) -> GetElementResponse: res = await self.send_request( - "actions/get-interaction-selector", data=dto.dict(), method="POST" + "actions/get-interaction-selector", data=dto.model_dump(), method="POST" ) return GetElementResponse(**res.json()) - async def make_interaction(self, dto: MakeInteractionDTO) -> InteractionResponse: + async def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: res = await self.send_request( - "actions/make-interaction", data=dto.dict(), method="POST" + "actions/make-interaction", data=dto.model_dump(), method="POST" ) res_dict = res.json() return InteractionResponse( status=res_dict["status"], message=res_dict["message"] ) - async def check_extract_cache(self, dto: ExtractDTO) -> CacheExtractResponse: - res = await self.send_request( - "actions/check-extract-cache", data=dto.dict(), method="POST" - ) - return CacheExtractResponse(**res.json()) - async def extract(self, dto: ExtractDTO) -> ExtractResponse: res = await self.send_request( - "actions/extract-page", data=dto.dict(), method="POST" + "actions/extract-page", data=dto.model_dump(), method="POST" ) res_dict = res.json() return ExtractResponse( @@ -91,7 +44,7 @@ async def extract(self, dto: ExtractDTO) -> ExtractResponse: async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: res = await self.send_request( - "actions/ask-page", data=dto.dict(), method="POST" + "actions/ask-page", data=dto.model_dump(), method="POST" ) res_dict = res.json() return AskPageResponse( @@ -99,22 +52,3 @@ async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: description=res_dict["description"], return_data=res_dict["return_data"], ) - - async def try_run_cached(self, dto: TryRunScriptDTO) -> Optional[ExtractResponse]: - res = await self.send_request( - "actions/try-run-cached", data=dto.dict(), method="POST" - ) - if res is None: - return None - res_dict = res.json() - loaded_value = res_dict["return_data"] - if loaded_value is None: - return None - - return ExtractResponse( - status=res_dict["status"], - message=res_dict["message"], - return_data=loaded_value, - created_script=res_dict.get("created_script", None), - used_cache=res_dict.get("used_cache", False), - ) diff --git a/dendrite/logic/hosted/_api/dto/__init__.py b/dendrite/logic/hosted/_api/dto/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/logic/hosted/_api/dto/ask_page_dto.py b/dendrite/logic/hosted/_api/dto/ask_page_dto.py deleted file mode 100644 index f2dcaf3..0000000 --- a/dendrite/logic/hosted/_api/dto/ask_page_dto.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Any, Optional -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class AskPageDTO(BaseModel): - prompt: str - return_schema: Optional[Any] - page_information: PageInformation - api_config: APIConfig diff --git a/dendrite/logic/hosted/_api/dto/authenticate_dto.py b/dendrite/logic/hosted/_api/dto/authenticate_dto.py deleted file mode 100644 index f5a1de7..0000000 --- a/dendrite/logic/hosted/_api/dto/authenticate_dto.py +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Union -from pydantic import BaseModel - - -class AuthenticateDTO(BaseModel): - domains: Union[str, list[str]] diff --git a/dendrite/logic/hosted/_api/dto/extract_dto.py b/dendrite/logic/hosted/_api/dto/extract_dto.py deleted file mode 100644 index 8cf1cc7..0000000 --- a/dendrite/logic/hosted/_api/dto/extract_dto.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -from typing import Any -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class ExtractDTO(BaseModel): - page_information: PageInformation - api_config: APIConfig - prompt: str - return_data_json_schema: Any - use_screenshot: bool = False - use_cache: bool = True - force_use_cache: bool = False - - @property - def combined_prompt(self) -> str: - - json_schema_prompt = ( - "" - if self.return_data_json_schema is None - else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" - ) - return f"Task: {self.prompt}{json_schema_prompt}" diff --git a/dendrite/logic/hosted/_api/dto/get_elements_dto.py b/dendrite/logic/hosted/_api/dto/get_elements_dto.py deleted file mode 100644 index 636c896..0000000 --- a/dendrite/logic/hosted/_api/dto/get_elements_dto.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Dict, Union -from pydantic import BaseModel - -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class CheckSelectorCacheDTO(BaseModel): - url: str - prompt: Union[str, Dict[str, str]] - - -class GetElementsDTO(BaseModel): - page_information: PageInformation - prompt: Union[str, Dict[str, str]] - api_config: APIConfig - use_cache: bool = True - only_one: bool - force_use_cache: bool = False diff --git a/dendrite/logic/hosted/_api/dto/get_interaction_dto.py b/dendrite/logic/hosted/_api/dto/get_interaction_dto.py deleted file mode 100644 index 93889c7..0000000 --- a/dendrite/logic/hosted/_api/dto/get_interaction_dto.py +++ /dev/null @@ -1,10 +0,0 @@ -from pydantic import BaseModel - -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class GetInteractionDTO(BaseModel): - page_information: PageInformation - api_config: APIConfig - prompt: str diff --git a/dendrite/logic/hosted/_api/dto/get_session_dto.py b/dendrite/logic/hosted/_api/dto/get_session_dto.py deleted file mode 100644 index 6414cc3..0000000 --- a/dendrite/logic/hosted/_api/dto/get_session_dto.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class GetSessionDTO(BaseModel): - user_id: str - domain: str diff --git a/dendrite/logic/hosted/_api/dto/google_search_dto.py b/dendrite/logic/hosted/_api/dto/google_search_dto.py deleted file mode 100644 index 6c55615..0000000 --- a/dendrite/logic/hosted/_api/dto/google_search_dto.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Optional -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation - - -class GoogleSearchDTO(BaseModel): - query: str - country: Optional[str] = None - filter_results_prompt: Optional[str] = None - page_information: PageInformation - api_config: APIConfig diff --git a/dendrite/logic/hosted/_api/dto/make_interaction_dto.py b/dendrite/logic/hosted/_api/dto/make_interaction_dto.py deleted file mode 100644 index c0592ad..0000000 --- a/dendrite/logic/hosted/_api/dto/make_interaction_dto.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Literal, Optional -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_diff_information import ( - PageDiffInformation, -) - - -InteractionType = Literal["click", "fill", "hover"] - - -class MakeInteractionDTO(BaseModel): - url: str - dendrite_id: str - interaction_type: InteractionType - value: Optional[str] = None - expected_outcome: Optional[str] - page_delta_information: PageDiffInformation - api_config: APIConfig diff --git a/dendrite/logic/hosted/_api/dto/try_run_script_dto.py b/dendrite/logic/hosted/_api/dto/try_run_script_dto.py deleted file mode 100644 index e283806..0000000 --- a/dendrite/logic/hosted/_api/dto/try_run_script_dto.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import Any, Optional -from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig - - -class TryRunScriptDTO(BaseModel): - url: str - raw_html: str - api_config: APIConfig - prompt: str - db_prompt: Optional[str] = ( - None # If you wish to cache a script based of a fixed prompt use this value - ) - return_data_json_schema: Any diff --git a/dendrite/logic/hosted/_api/dto/upload_auth_session_dto.py b/dendrite/logic/hosted/_api/dto/upload_auth_session_dto.py deleted file mode 100644 index 1697fdf..0000000 --- a/dendrite/logic/hosted/_api/dto/upload_auth_session_dto.py +++ /dev/null @@ -1,11 +0,0 @@ -from pydantic import BaseModel - -from dendrite.browser.async_api._core.models.authentication import ( - AuthSession, - StorageState, -) - - -class UploadAuthSessionDTO(BaseModel): - auth_data: AuthSession - storage_state: StorageState diff --git a/dendrite/logic/hosted/_api/response/__init__.py b/dendrite/logic/hosted/_api/response/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/logic/hosted/_api/response/cache_extract_response.py b/dendrite/logic/hosted/_api/response/cache_extract_response.py deleted file mode 100644 index 463d03b..0000000 --- a/dendrite/logic/hosted/_api/response/cache_extract_response.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class CacheExtractResponse(BaseModel): - exists: bool diff --git a/dendrite/logic/hosted/_api/response/google_search_response.py b/dendrite/logic/hosted/_api/response/google_search_response.py deleted file mode 100644 index d435b71..0000000 --- a/dendrite/logic/hosted/_api/response/google_search_response.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class SearchResult(BaseModel): - url: str - title: str - description: str - - -class GoogleSearchResponse(BaseModel): - results: List[SearchResult] diff --git a/dendrite/logic/hosted/_api/response/interaction_response.py b/dendrite/logic/hosted/_api/response/interaction_response.py deleted file mode 100644 index 3dd2e49..0000000 --- a/dendrite/logic/hosted/_api/response/interaction_response.py +++ /dev/null @@ -1,7 +0,0 @@ -from pydantic import BaseModel -from dendrite.browser.async_api._common.status import Status - - -class InteractionResponse(BaseModel): - message: str - status: Status diff --git a/dendrite/logic/hosted/_api/response/selector_cache_response.py b/dendrite/logic/hosted/_api/response/selector_cache_response.py deleted file mode 100644 index 4c0e388..0000000 --- a/dendrite/logic/hosted/_api/response/selector_cache_response.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class SelectorCacheResponse(BaseModel): - exists: bool diff --git a/dendrite/logic/hosted/_api/response/session_response.py b/dendrite/logic/hosted/_api/response/session_response.py deleted file mode 100644 index 2d03b97..0000000 --- a/dendrite/logic/hosted/_api/response/session_response.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import List -from pydantic import BaseModel - - -class SessionResponse(BaseModel): - cookies: List[dict] - origins_storage: List[dict] diff --git a/dendrite/logic/interfaces/async_api.py b/dendrite/logic/interfaces/async_api.py index 663e37f..0ebbe0f 100644 --- a/dendrite/logic/interfaces/async_api.py +++ b/dendrite/logic/interfaces/async_api.py @@ -1,48 +1,44 @@ -# dendrite/browser/async_api/_api/protocols.py -from typing import Protocol, Optional -from dendrite.browser.async_api._api.dto.authenticate_dto import AuthenticateDTO -from dendrite.browser.async_api._api.dto.upload_auth_session_dto import UploadAuthSessionDTO -from dendrite.browser.async_api._api.dto.get_elements_dto import ( - GetElementsDTO, - CheckSelectorCacheDTO, -) -from dendrite.browser.async_api._api.dto.make_interaction_dto import MakeInteractionDTO -from dendrite.browser.async_api._api.dto.extract_dto import ExtractDTO -from dendrite.browser.async_api._api.dto.ask_page_dto import AskPageDTO -from dendrite.browser.async_api._api.dto.try_run_script_dto import TryRunScriptDTO -from dendrite.browser.async_api._api.response.selector_cache_response import SelectorCacheResponse -from dendrite.browser.async_api._api.response.get_element_response import GetElementResponse -from dendrite.browser.async_api._api.response.interaction_response import InteractionResponse -from dendrite.browser.async_api._api.response.extract_response import ExtractResponse -from dendrite.browser.async_api._api.response.ask_page_response import AskPageResponse -from dendrite.browser.async_api._api.response.cache_extract_response import CacheExtractResponse + +from typing import Protocol + from dendrite.browser.async_api._core.models.authentication import AuthSession +from dendrite.logic.get_element import get_element +from dendrite.models.dto.ask_page_dto import AskPageDTO +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.dto.get_elements_dto import CheckSelectorCacheDTO, GetElementsDTO +from dendrite.models.dto.make_interaction_dto import VerifyActionDTO +from dendrite.models.response.ask_page_response import AskPageResponse +from dendrite.models.response.extract_response import ExtractResponse +from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.response.interaction_response import InteractionResponse -class BrowserAPIProtocol(Protocol): - async def authenticate(self, dto: AuthenticateDTO) -> AuthSession: - ... +from dendrite.logic.ask import ask +from dendrite.logic.extract import extract +from dendrite.logic import verify_interaction - async def upload_auth_session(self, dto: UploadAuthSessionDTO) -> None: - ... +class LogicAPIProtocol(Protocol): - async def check_selector_cache(self, dto: CheckSelectorCacheDTO) -> SelectorCacheResponse: - ... + async def get_element( + self, dto: GetElementsDTO + ) -> GetElementResponse: ... - async def get_interactions_selector(self, dto: GetElementsDTO) -> GetElementResponse: - ... + async def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: ... - async def make_interaction(self, dto: MakeInteractionDTO) -> InteractionResponse: - ... + async def extract(self, dto: ExtractDTO) -> ExtractResponse: ... - async def check_extract_cache(self, dto: ExtractDTO) -> CacheExtractResponse: - ... + async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... - async def extract(self, dto: ExtractDTO) -> ExtractResponse: - ... +class LocalProtocol(LogicAPIProtocol): + async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: + return await get_element.get_element(dto) + + async def extract(self, dto: ExtractDTO) -> ExtractResponse: + return await extract.extract(dto) + + async def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: + return await verify_interaction.verify_action(dto) + async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: - ... - - async def try_run_cached(self, dto: TryRunScriptDTO) -> Optional[ExtractResponse]: - ... \ No newline at end of file + return await ask.ask_page_action(dto) diff --git a/dendrite/logic/interfaces/cache.py b/dendrite/logic/interfaces/cache.py index 46ba571..aa368d6 100644 --- a/dendrite/logic/interfaces/cache.py +++ b/dendrite/logic/interfaces/cache.py @@ -1,24 +1,5 @@ +from typing import Generic, Protocol, TypeVar, Union, overload -from typing import Protocol, Union, overload - -from typing import Protocol, TypeVar, Generic from pydantic import BaseModel -T = TypeVar('T', bound=BaseModel) - -class CacheProtocol(Protocol, Generic[T]): - - @overload - def get(self, key: dict) -> Union[T, None]: - ... - @overload - def get(self, key: str) -> Union[T, None]: - ... - def get(self, key: Union[str,dict]) -> Union[T, None]: - ... - - def set(self, key: str, value: T) -> None: - ... - - def delete(self, key: str) -> None: - ... \ No newline at end of file +T = TypeVar("T", bound=BaseModel) diff --git a/dendrite/logic/interfaces/sync_api.py b/dendrite/logic/interfaces/sync_api.py new file mode 100644 index 0000000..3eafa85 --- /dev/null +++ b/dendrite/logic/interfaces/sync_api.py @@ -0,0 +1,75 @@ + +import asyncio +from concurrent.futures import ThreadPoolExecutor +import threading +from typing import Any, Coroutine, Protocol, TypeVar + +from dendrite.browser.async_api._core.models.authentication import AuthSession +from dendrite.logic.get_element import get_element +from dendrite.models.dto.ask_page_dto import AskPageDTO +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.dto.get_elements_dto import CheckSelectorCacheDTO, GetElementsDTO +from dendrite.models.dto.make_interaction_dto import VerifyActionDTO +from dendrite.models.response.ask_page_response import AskPageResponse + +from dendrite.models.response.extract_response import ExtractResponse +from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.response.interaction_response import InteractionResponse + +from dendrite.logic.ask import ask +from dendrite.logic.extract import extract +from dendrite.logic import verify_interaction + + +T = TypeVar("T") + + +def run_coroutine_sync(coroutine: Coroutine[Any, Any, T], timeout: float = 30) -> T: + def run_in_new_loop(): + new_loop = asyncio.new_event_loop() + asyncio.set_event_loop(new_loop) + try: + return new_loop.run_until_complete(coroutine) + finally: + new_loop.close() + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coroutine) + + if threading.current_thread() is threading.main_thread(): + if not loop.is_running(): + return loop.run_until_complete(coroutine) + else: + with ThreadPoolExecutor() as pool: + future = pool.submit(run_in_new_loop) + return future.result(timeout=timeout) + else: + return asyncio.run_coroutine_threadsafe(coroutine, loop).result() + +class LogicAPIProtocol(Protocol): + + def get_element( + self, dto: GetElementsDTO + ) -> GetElementResponse: ... + + def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: ... + + def extract(self, dto: ExtractDTO) -> ExtractResponse: ... + + def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... + + +class LocalProtocol(LogicAPIProtocol): + def get_element(self, dto: GetElementsDTO) -> GetElementResponse: + return run_coroutine_sync(get_element.get_element(dto)) + + def extract(self, dto: ExtractDTO) -> ExtractResponse: + return run_coroutine_sync(extract.extract(dto)) + + def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: + return run_coroutine_sync(verify_interaction.verify_action(dto)) + + def ask_page(self, dto: AskPageDTO) -> AskPageResponse: + return run_coroutine_sync(ask.ask_page_action(dto)) diff --git a/dendrite/logic/llm/agent.py b/dendrite/logic/llm/agent.py new file mode 100644 index 0000000..39c5298 --- /dev/null +++ b/dendrite/logic/llm/agent.py @@ -0,0 +1,233 @@ +from typing import Any, Dict, List, Optional, Union, cast + +import litellm +from litellm.files.main import ModelResponse +from loguru import logger +from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam + +Message = ChatCompletionMessageParam + + +class LLMContextLengthExceededException(Exception): + CONTEXT_LIMIT_ERRORS = [ + "expected a string with maximum length", + "maximum context length", + "context length exceeded", + "context_length_exceeded", + "context window full", + "too many tokens", + "input is too long", + "exceeds token limit", + ] + + def __init__(self, error_message: str): + self.original_error_message = error_message + super().__init__(self._get_error_message(error_message)) + + def _is_context_limit_error(self, error_message: str) -> bool: + return any( + phrase.lower() in error_message.lower() + for phrase in self.CONTEXT_LIMIT_ERRORS + ) + + def _get_error_message(self, error_message: str): + return ( + f"LLM context length exceeded. Original error: {error_message}\n" + "Consider using a smaller input or implementing a text splitting strategy." + ) + + +LLM_CONTEXT_WINDOW_SIZES = { + # openai + "gpt-4": 8192, + "gpt-4o": 128000, + "gpt-4o-mini": 128000, + "gpt-4-turbo": 128000, + "o1-preview": 128000, + "o1-mini": 128000, + # deepseek + "deepseek-chat": 128000, + # groq + "gemma2-9b-it": 8192, + "gemma-7b-it": 8192, + "llama3-groq-70b-8192-tool-use-preview": 8192, + "llama3-groq-8b-8192-tool-use-preview": 8192, + "llama-3.1-70b-versatile": 131072, + "llama-3.1-8b-instant": 131072, + "llama-3.2-1b-preview": 8192, + "llama-3.2-3b-preview": 8192, + "llama-3.2-11b-text-preview": 8192, + "llama-3.2-90b-text-preview": 8192, + "llama3-70b-8192": 8192, + "llama3-8b-8192": 8192, + "mixtral-8x7b-32768": 32768, +} + + +class LLM: + def __init__( + self, + model: str, + timeout: Optional[Union[float, int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + max_completion_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, + presence_penalty: Optional[float] = None, + frequency_penalty: Optional[float] = None, + logit_bias: Optional[Dict[int, float]] = None, + response_format: Optional[Dict[str, Any]] = None, + seed: Optional[int] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, + base_url: Optional[str] = None, + api_version: Optional[str] = None, + api_key: Optional[str] = None, + callbacks: List[Any] = [], + **kwargs, + ): + self.model = model + self.timeout = timeout + self.temperature = temperature + self.top_p = top_p + self.n = n + self.stop = stop + self.max_completion_tokens = max_completion_tokens + self.max_tokens = max_tokens + self.presence_penalty = presence_penalty + self.frequency_penalty = frequency_penalty + self.logit_bias = logit_bias + self.response_format = response_format + self.seed = seed + self.logprobs = logprobs + self.top_logprobs = top_logprobs + self.base_url = base_url + self.api_version = api_version + self.api_key = api_key + self.callbacks = callbacks + self.kwargs = kwargs + + litellm.drop_params = True + + def call(self, messages: Message) -> str: + + try: + params = { + "model": self.model, + "messages": messages, + "timeout": self.timeout, + "temperature": self.temperature, + "top_p": self.top_p, + "n": self.n, + "stop": self.stop, + "max_tokens": self.max_tokens or self.max_completion_tokens, + "presence_penalty": self.presence_penalty, + "frequency_penalty": self.frequency_penalty, + "logit_bias": self.logit_bias, + "response_format": self.response_format, + "seed": self.seed, + "logprobs": self.logprobs, + "top_logprobs": self.top_logprobs, + "api_base": self.base_url, + "api_version": self.api_version, + "api_key": self.api_key, + "stream": False, + **self.kwargs, + } + + params = {k: v for k, v in params.items() if v is not None} + + response = litellm.completion(**params) + response = cast(ModelResponse, response) + return response["choices"][0]["message"]["content"] + except Exception as e: + if not LLMContextLengthExceededException(str(e))._is_context_limit_error( + str(e) + ): + logger.error(f"LiteLLM call failed: {str(e)}") + + raise # Re-raise the exception after logging + + async def acall(self, messages: List[Message]) -> ModelResponse: + + try: + params = { + "model": self.model, + "messages": messages, + "timeout": self.timeout, + "temperature": self.temperature, + "top_p": self.top_p, + "n": self.n, + "stop": self.stop, + "max_tokens": self.max_tokens or self.max_completion_tokens, + "presence_penalty": self.presence_penalty, + "frequency_penalty": self.frequency_penalty, + "logit_bias": self.logit_bias, + "response_format": self.response_format, + "seed": self.seed, + "logprobs": self.logprobs, + "top_logprobs": self.top_logprobs, + "api_base": self.base_url, + "api_version": self.api_version, + "api_key": self.api_key, + "stream": False, + **self.kwargs, + } + + params = {k: v for k, v in params.items() if v is not None} + + response = await litellm.acompletion(**params) + response = cast(ModelResponse, response) + return response + except Exception as e: + if not LLMContextLengthExceededException(str(e))._is_context_limit_error( + str(e) + ): + logger.error(f"LiteLLM call failed: {str(e)}") + + raise # Re-raise the exception after logging + + def get_context_window_size(self) -> int: + return int(LLM_CONTEXT_WINDOW_SIZES.get(self.model, 8192) * 0.75) + + +class Agent: + def __init__( + self, + model: Union[LLM, str], + system_prompt: Optional[str] = None, + ): + self.messages: List[Message] = ( + [] if not system_prompt else [{"role": "system", "content": system_prompt}] + ) + + if isinstance(model, str): + self.llm = LLM(model) + else: + self.llm = model + + async def add_message(self, message: str) -> str: + self.messages.append({"role": "user", "content": message}) + + text = await self.call_llm(self.messages) + + self.messages.append({"role": "assistant", "content": text}) + + return text + + async def call_llm(self, messages: List[Message]) -> str: + res = await self.llm.acall(messages) + + if len(res.choices) == 0: + logger.error("No choices outputed: ", res) + raise Exception("No choices from model") + + choices = cast(List[litellm.Choices], res.choices) + text = choices[0].message.content + + if text is None: + logger.error("No text content in the response") + raise Exception("No text content in the response") + return text diff --git a/dendrite/logic/llm/config.py b/dendrite/logic/llm/config.py new file mode 100644 index 0000000..8471716 --- /dev/null +++ b/dendrite/logic/llm/config.py @@ -0,0 +1,87 @@ +from typing import Dict, Literal, Optional, overload + +from dendrite.logic.llm.agent import LLM + +try: + import tomllib # type: ignore +except ModuleNotFoundError: + import tomli as tomllib # type: ignore # tomllib is only included standard lib for python 3.11+ + + +DEFAULT_LLM = { + "extract_agent": LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500), + "scroll_agent": LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500), + "ask_page_agent": LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500), + "segment_agent": LLM("gpt-4o", temperature=0, max_tokens=1500), + "select_agent": LLM("claude-3-5-sonnet-20241022", temperature=0, max_tokens=1500), + "verify_action_agent": LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500), +} + +class LLMConfig(): + def __init__(self, default_agents: Optional[Dict[str, LLM]] = None, default_llm: Optional[LLM] = None): + self.registered_llms: Dict[str, LLM] = DEFAULT_LLM.copy() + if default_agents: + self.registered_llms.update(default_agents) + + self.default_llm = default_llm or LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500) + + async def register_agent(self, agent: str, llm: LLM) -> None: + """ + Register an LLM agent by name. + + Args: + agent: The name of the agent to register + llm: The LLM agent to register + """ + self.registered_llms[agent] = llm + + async def register(self, agents: Dict[str, LLM]) -> None: + """ + Register multiple LLM agents at once. Overrides if an agent has already been registered + + Args: + agents: A dictionary of agent names to LLM agents + """ + self.registered_llms.update(agents) + + @overload + def get(self, agent: str) -> LLM: ... + + @overload + def get(self, agent: str, default: LLM) -> LLM: ... + + @overload + def get(self, agent: str, default: Optional[LLM] = ..., use_default: Literal[False] = False) -> Optional[LLM]: ... + + def get( + self, + agent: str, + default: Optional[LLM] = None, + use_default: bool = True, + ) -> Optional[LLM]: + """ + Get an LLM agent by name, optionally falling back to default if not found. + + Args: + agent: The name of the agent to retrieve + default: Optional specific default LLM to use if agent not found + use_default: If True, use self.default_llm when agent not found and default is None + + Returns: + Optional[LLM]: The requested LLM agent, default LLM, or None + """ + llm = self.registered_llms.get(agent) + if llm is not None: + return llm + + if default is not None: + return default + + if use_default and self.default_llm is not None: + return self.default_llm + + return None + + +# Create a single instance +llm_config = LLMConfig() diff --git a/dendrite/logic/local/llm/token_count.py b/dendrite/logic/llm/token_count.py similarity index 67% rename from dendrite/logic/local/llm/token_count.py rename to dendrite/logic/llm/token_count.py index 12e6d80..c7f57fb 100644 --- a/dendrite/logic/local/llm/token_count.py +++ b/dendrite/logic/llm/token_count.py @@ -1,7 +1,7 @@ import tiktoken -def token_count(string: str, encoding_name) -> int: +def token_count(string: str, encoding_name: str = "gpt-4o") -> int: encoding = tiktoken.encoding_for_model(encoding_name) num_tokens = len(encoding.encode(string)) return num_tokens diff --git a/dendrite/logic/local/code/execute.py b/dendrite/logic/local/code/execute.py deleted file mode 100644 index 78720ed..0000000 --- a/dendrite/logic/local/code/execute.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing import Any - -from bs4 import BeautifulSoup -from .code_session import CodeSession - - -def execute(script: str, raw_html: str, return_data_json_schema) -> Any: - code_session = CodeSession() - soup = BeautifulSoup(raw_html, "lxml") - try: - - created_variables = code_session.exec_code(script, soup, raw_html) - - if "response_data" in created_variables: - response_data = created_variables["response_data"] - - try: - code_session.validate_response(return_data_json_schema, response_data) - except Exception as e: - raise Exception(f"Failed to validate response data. Exception: {e}") - - return response_data - else: - raise Exception("No return data available for this script.") - except Exception as e: - raise e diff --git a/dendrite/logic/local/dom/strip.py b/dendrite/logic/local/dom/strip.py deleted file mode 100644 index 54050fb..0000000 --- a/dendrite/logic/local/dom/strip.py +++ /dev/null @@ -1,52 +0,0 @@ -from bs4 import BeautifulSoup, Doctype, Tag, Comment - - -def mild_strip(soup: Tag, keep_d_id: bool = True) -> BeautifulSoup: - new_soup = BeautifulSoup(str(soup), "html.parser") - _mild_strip(new_soup, keep_d_id) - return new_soup - - -def mild_strip_in_place(soup: BeautifulSoup, keep_d_id: bool = True) -> None: - _mild_strip(soup, keep_d_id) - - -def _mild_strip(soup: BeautifulSoup, keep_d_id: bool = True) -> None: - for element in soup(text=lambda text: isinstance(text, Comment)): - element.extract() - - # for text in soup.find_all(text=lambda text: isinstance(text, NavigableString)): - # if len(text) > 200: - # text.replace_with(text[:200] + f"... [{len(text)-200} more chars]") - - for tag in soup( - ["head", "script", "style", "path", "polygon", "defs", "svg", "br", "Doctype"] - ): - tag.extract() - - for element in soup.contents: - if isinstance(element, Doctype): - element.extract() - - # for tag in soup.find_all(True): - # tag.attrs = { - # attr: (value[:100] if isinstance(value, str) else value) - # for attr, value in tag.attrs.items() - # } - # if keep_d_id == False: - # del tag["d-id"] - for tag in soup.find_all(True): - if tag.attrs.get("is-interactable-d_id") == "true": - continue - - tag.attrs = { - attr: (value[:100] if isinstance(value, str) else value) - for attr, value in tag.attrs.items() - } - if keep_d_id == False: - del tag["d-id"] - - # if browser != None: - # for elem in list(soup.descendants): - # if isinstance(elem, Tag) and not browser.element_is_visible(elem): - # elem.extract() diff --git a/dendrite/logic/local/extract/extract_agent.py b/dendrite/logic/local/extract/extract_agent.py deleted file mode 100644 index 28e87e7..0000000 --- a/dendrite/logic/local/extract/extract_agent.py +++ /dev/null @@ -1,529 +0,0 @@ -import json -import re -from typing import Any, Optional -from anthropic.types import TextBlock -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.logic.local.dom.strip import mild_strip -from dendrite.logic.local.extract.prompts import create_script_prompt_segmented_html -from dendrite.logic.local.extract.scroll_agent import ScrollAgent -from dendrite.logic.local.get_element.hanifi_search import get_expanded_dom -from dendrite.models.dto.extract_dto import ExtractDTO -from dendrite.models.page_information import PageInformation - -from bs4 import BeautifulSoup, Tag - -from dendrite.models.response.extract_page_response import ExtractPageResponse -from ..code.code_session import CodeSession -from ..ask.image import segment_image -from dendrite_server_merge.core.llm.claude import async_claude_request - -from dendrite_server_merge.logging import agent_logger - -from loguru import logger - - - -class ExtractAgent: - def __init__( - self, page_information: PageInformation, api_config: APIConfig, user_id: str - ) -> None: - self.page_information = page_information - self.soup = BeautifulSoup(page_information.raw_html, "lxml") - self.api_config = api_config - self.messages = [] - self.generated_script: Optional[str] = None - self.user_id = user_id - self.scroll_agent = ScrollAgent(api_config, page_information) - - def get_generated_script(self): - return self.generated_script - - async def write_and_run_script( - self, extract_page_dto: ExtractDTO - ) -> ExtractPageResponse: - mild_soup = mild_strip(self.soup) - - search_terms = [] - - segments = segment_image( - extract_page_dto.page_information.screenshot_base64, segment_height=4000 - ) - - scroll_result = await self.scroll_agent.scroll_through_page( - extract_page_dto.combined_prompt, - image_segments=segments, - ) - - if scroll_result.status == "error": - return ExtractPageResponse( - status="impossible", - message=str(scroll_result.message), - return_data=None, - used_cache=False, - created_script=None, - ) - - if scroll_result.status == "loading": - return ExtractPageResponse( - status="loading", - message="This page is still loading. Please wait a bit longer.", - return_data=None, - used_cache=False, - created_script=None, - ) - - expanded_html = None - if scroll_result.element_to_inspect_html: - combined_prompt = ( - "Get these elements (make sure you only return element that you are confident that these are the correct elements, it's OK to not select any elements):\n- " - + "\n- ".join(scroll_result.element_to_inspect_html) - ) - expanded = await get_expanded_dom( - mild_soup, - combined_prompt, - self.api_config, - ) - if expanded: - expanded_html = expanded[0] - - if expanded_html: - return await self.code_script_from_found_expanded_html_tags( - extract_page_dto, expanded_html, segments - ) - else: - compress_html = CompressHTML( - mild_soup, - exclude_dendrite_ids=False, - focus_on_text=True, - max_token_size=16000, - max_size_per_element=10000, - compression_multiplier=0.5, - ) - expanded_html = await compress_html.compress(search_terms) - return await self.code_script_from_compressed_html( - extract_page_dto, expanded_html, segments, mild_soup - ) - - def segment_large_tag(self, tag): - segments = [] - current_segment = "" - current_tokens = 0 - for line in tag.split("\n"): - line_tokens = token_count(line) - if current_tokens + line_tokens > 4000: - segments.append(current_segment) - current_segment = line - current_tokens = line_tokens - else: - current_segment += line + "\n" - current_tokens += line_tokens - if current_segment: - segments.append(current_segment) - return segments - - async def code_script_from_found_expanded_html_tags( - self, extract_page_dto: ExtractDTO, expanded_html, segments - ): - agent_logger.info("Starting code_script_from_found_expanded_html_tags method") - messages = [] - - user_prompt = create_script_prompt_segmented_html( - extract_page_dto.combined_prompt, - expanded_html, - self.page_information.url, - ) - agent_logger.debug(f"User prompt created: {user_prompt[:100]}...") - - content = [ - { - "type": "text", - "text": user_prompt, - }, - ] - - messages = [ - {"role": "user", "content": content}, - ] - - iterations = 0 - max_retries = 10 - - generated_script: str = "" - response_data: Any | None = None - - while iterations <= max_retries: - iterations += 1 - agent_logger.info(f"Starting iteration {iterations}") - - config = { - "messages": messages, - "model": "claude-3-5-sonnet-20241022", - "temperature": 0.3, - "max_tokens": 1500, - } - res = await async_claude_request(config, self.api_config) - if not isinstance(res.content[0], TextBlock): - logger.error("Needs to be an text block: ", res) - raise Exception("Needs to be an text block") - - text = res.content[0].text - dict_res = { - "role": "assistant", - "content": text, - } - messages.append(dict_res) - - json_pattern = r"```json(.*?)```" - code_pattern = r"```python(.*?)```" - - if text: - json_matches = re.findall(json_pattern, text, re.DOTALL) - code_matches = re.findall(code_pattern, text, re.DOTALL) - - if len(json_matches) + len(code_matches) > 1: - content = "Error: Please output only one action at a time (either JSON or Python code, not both)." - messages.append({"role": "user", "content": content}) - continue - - for code_match in code_matches: - agent_logger.debug("Processing code match") - generated_script = code_match.strip() - temp_code_session = CodeSession() - try: - variables = temp_code_session.exec_code( - generated_script, - self.soup, - self.page_information.raw_html, - ) - agent_logger.debug("Code execution successful") - except Exception as e: - agent_logger.error(f"Code execution failed: {str(e)}") - content = f"Error: {str(e)}" - messages.append({"role": "user", "content": content}) - continue - - try: - if "response_data" in variables: - response_data = variables["response_data"] - # agent_logger.debug(f"Response data: {response_data}") - - if extract_page_dto.return_data_json_schema != None: - temp_code_session.validate_response( - extract_page_dto.return_data_json_schema, - response_data, - ) - - llm_readable_exec_res = ( - temp_code_session.llm_readable_exec_res( - variables, - extract_page_dto.combined_prompt, - iterations, - max_retries, - ) - ) - - messages.append( - {"role": "user", "content": llm_readable_exec_res} - ) - continue - else: - content = ( - f"Error: You need to add the variable 'response_data'" - ) - messages.append( - { - "role": "user", - "content": content, - } - ) - continue - except Exception as e: - llm_readable_exec_res = temp_code_session.llm_readable_exec_res( - variables, - extract_page_dto.combined_prompt, - iterations, - max_retries, - ) - content = f"Error: Failed to validate `response_data`. Exception: {e}. {llm_readable_exec_res}" - messages.append( - { - "role": "user", - "content": content, - } - ) - continue - - for json_match in json_matches: - agent_logger.debug("Processing JSON match") - extracted_json = json_match.strip() - data_dict = json.loads(extracted_json) - current_segment = 0 - if "request_more_html" in data_dict: - agent_logger.info("Processing element indexes") - try: - current_segment += 1 - content = f"""Here is more of the HTML:\n```html\n{expanded_html[LARGE_HTML_CHAR_TRUNCATE_LEN*current_segment:LARGE_HTML_CHAR_TRUNCATE_LEN*(current_segment+1)]}\n```""" - if len(expanded_html) > LARGE_HTML_CHAR_TRUNCATE_LEN * ( - current_segment + 1 - ): - content += "\nThere is still more HTML to see. You can request more if needed." - else: - content += "\nThis is the end of the HTML content." - messages.append({"role": "user", "content": content}) - continue - except Exception as e: - agent_logger.error( - f"Error processing element indexes: {str(e)}" - ) - content = f"Error: {str(e)}" - messages.append({"role": "user", "content": content}) - continue - elif "error" in data_dict: - agent_logger.error(f"Error in data_dict: {data_dict['error']}") - raise HTTPException(404, detail=data_dict["error"]) - elif "success" in data_dict: - agent_logger.info("Script generation successful") - self.generated_script = generated_script - - await upsert_script_in_db( - extract_page_dto.combined_prompt, - generated_script, - extract_page_dto.page_information.url, - user_id=self.user_id, - ) - # agent_logger.debug(f"Response data: {response_data}") - return ExtractPageResponse( - status="success", - message=data_dict["success"], - return_data=response_data, - used_cache=False, - created_script=self.get_generated_script(), - ) - - agent_logger.warning("Failed to create script after retrying several times") - return ExtractPageResponse( - status="failed", - message="Failed to create script after retrying several times.", - return_data=None, - used_cache=False, - created_script=self.get_generated_script(), - ) - - async def code_script_from_compressed_html( - self, extract_page_dto: ExtractDTO, expanded_html, segments, mild_soup - ): - messages = [] - - user_prompt = create_script_prompt_compressed_html( - extract_page_dto.combined_prompt, - expanded_html, - self.page_information.url, - ) - - content = [ - { - "type": "text", - "text": user_prompt, - }, - ] - - if extract_page_dto.use_screenshot: - content += [ - { - "type": "text", - "text": "Here is a screenshot of the website:", - }, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": segments[0], - }, - }, - ] - - messages = [ - {"role": "user", "content": content}, - ] - - iterations = 0 - max_retries = 10 - - generated_script: str = "" - response_data: Any | None = None - - while iterations <= max_retries: - iterations += 1 - - config = { - "messages": messages, - "model": "claude-3-5-sonnet-20241022", - "temperature": 0.3, - "max_tokens": 1500, - } - res = await async_claude_request(config, self.api_config) - if not isinstance(res.content[0], TextBlock): - logger.error("Needs to be an text block: ", res) - raise Exception("Needs to be an text block") - - text = res.content[0].text - dict_res = { - "role": "assistant", - "content": text, - } - messages.append(dict_res) - - json_pattern = r"```json(.*?)```" - code_pattern = r"```python(.*?)```" - - if text: - json_matches = re.findall(json_pattern, text, re.DOTALL) - code_matches = re.findall(code_pattern, text, re.DOTALL) - - if len(json_matches) + len(code_matches) > 1: - content = "Error: Please output only one action at a time (either JSON or Python code, not both)." - messages.append({"role": "user", "content": content}) - continue - - for code_match in code_matches: - generated_script = code_match.strip() - temp_code_session = CodeSession() - try: - variables = temp_code_session.exec_code( - generated_script, - self.soup, - self.page_information.raw_html, - ) - except Exception as e: - content = f"Error: {str(e)}" - messages.append({"role": "user", "content": content}) - continue - - try: - if "response_data" in variables: - response_data = variables["response_data"] - # agent_logger.debug(f"Response data: {response_data}") - - if extract_page_dto.return_data_json_schema != None: - temp_code_session.validate_response( - extract_page_dto.return_data_json_schema, - response_data, - ) - - llm_readable_exec_res = ( - temp_code_session.llm_readable_exec_res( - variables, - extract_page_dto.combined_prompt, - iterations, - max_retries, - ) - ) - - messages.append( - {"role": "user", "content": llm_readable_exec_res} - ) - continue - else: - content = ( - f"Error: You need to add the variable 'response_data'" - ) - messages.append( - { - "role": "user", - "content": content, - } - ) - continue - except Exception as e: - llm_readable_exec_res = temp_code_session.llm_readable_exec_res( - variables, - extract_page_dto.combined_prompt, - iterations, - max_retries, - ) - content = f"Error: Failed to validate `response_data`. Exception: {e}. {llm_readable_exec_res}" - messages.append( - { - "role": "user", - "content": content, - } - ) - continue - - for json_match in json_matches: - extracted_json = json_match.strip() - data_dict = json.loads(extracted_json) - content = "" - if "d-ids" in data_dict: - try: - - content += "Here is the expanded HTML:" - for d_id in data_dict["d-ids"]: - d_id_res = mild_soup.find(attrs={"d-id": d_id}) - - tag = None - - if isinstance(d_id_res, Tag): - tag = d_id_res - - if tag: - subsection_mild = mild_strip(tag) - pretty = subsection_mild.prettify() - if len(pretty) > 120000: - compress_html = CompressHTML( - subsection_mild, - exclude_dendrite_ids=False, - max_token_size=16000, - max_size_per_element=10000, - focus_on_text=True, - compression_multiplier=0.3, - ) - subsection_compressed_html = ( - await compress_html.compress() - ) - content += f"\n\nThis expanded element with the d-id '{d_id}' was too large to inspect fully! Here is a compressed version of the element you selected, please inspect a smaller section of it:\n```html\n{subsection_compressed_html}\n```" - else: - subsection_mild = mild_strip( - tag, keep_d_id=False - ) - pretty = subsection_mild.prettify() - content += f"\n\nExpanded element with the d-id '{d_id}':\n```html\n{pretty}\n```" - else: - content += f"\n\nNo valid element could be found with the d-id or id '{d_id}'. Prefer using the d-id attribute." - - content += "\n\nIf you cannot find the relevant data in this HTML, consider expanding a different region." - messages.append({"role": "user", "content": content}) - continue - except Exception as e: - messages.append( - {"role": "user", "content": f"Error: {str(e)}"} - ) - agent_logger.debug(f"role: user, content: Error: {str(e)}") - elif "error" in data_dict: - raise HTTPException(404, detail=data_dict["error"]) - elif "success" in data_dict: - self.generated_script = generated_script - - await upsert_script_in_db( - extract_page_dto.combined_prompt, - generated_script, - extract_page_dto.page_information.url, - user_id=self.user_id, - ) - - return ExtractPageResponse( - status="success", - message=data_dict["success"], - return_data=response_data, - used_cache=False, - created_script=self.get_generated_script(), - ) - - return ExtractPageResponse( - status="failed", - message="Failed to create script after retrying several times.", - return_data=None, - used_cache=False, - created_script=self.get_generated_script(), - ) diff --git a/dendrite/logic/local/get_element/agents/agent.py b/dendrite/logic/local/get_element/agents/agent.py deleted file mode 100644 index cfa1e0b..0000000 --- a/dendrite/logic/local/get_element/agents/agent.py +++ /dev/null @@ -1,146 +0,0 @@ -from abc import ABC, abstractmethod -import json -from typing import Any, Dict, Generic, List, Literal, Optional, Type, TypeVar - -from anthropic.types import Message, TextBlock - -from dendrite_server_merge.core.llm.claude import async_claude_request -from dendrite_server_merge.core.llm.gemini import async_gemini_request -from dendrite_server_merge.core.llm.openai import async_openai_request -from dendrite_server_merge.models.APIConfig import APIConfig - - -T = TypeVar("T") -U = TypeVar("U") - - -class Agent(ABC, Generic[T, U]): - def __init__( - self, - model: U, - api_config: APIConfig, - system_message: Optional[str] = None, - temperature: float = 0, - max_tokens: int = 1500, - ): - self.messages: List[Dict] = [] - self.api_config = api_config - self.model = model - self.temperature = temperature - self.max_tokens = max_tokens - - if system_message: - self._add_system_message(system_message) - - @abstractmethod - def _add_system_message(self, message: str) -> None: - pass - - @abstractmethod - async def add_message(self, message: str) -> T: - pass - - -class AnthropicAgent( - Agent[Message, Literal["claude-3-5-sonnet-20241022", "claude-3-haiku-20240307"]] -): - def __init__( - self, - model: Literal["claude-3-5-sonnet-20241022", "claude-3-haiku-20240307"], - api_config: APIConfig, - system_message: Optional[str] = None, - temperature: float = 0, - max_tokens: int = 1500, - enable_caching: bool = False, # Add enable_caching option - ): - self.enable_caching = enable_caching # Store the caching preference - super().__init__(model, api_config, system_message, temperature, max_tokens) - - def _add_system_message(self, message: str) -> None: - self.system_msg: List[dict] = [{"type": "text", "text": message}] - if self.enable_caching: - self.system_msg[0]["cache_control"] = {"type": "ephemeral"} - - async def add_message(self, message: str) -> Message: - self.messages.append({"role": "user", "content": message}) - - spec = { - "messages": self.messages, - "model": self.model, - "temperature": self.temperature, - "max_tokens": self.max_tokens, - } - - if self.system_msg: - spec["system"] = self.system_msg - - res = await async_claude_request( - spec, self.api_config, enable_caching=self.enable_caching - ) - - if isinstance(res.content[0], TextBlock): - self.messages.append({"role": "assistant", "content": res.content[0].text}) - else: - raise ValueError("Unexpected response type: ", type(res.content[0])) - - return res - - def dump_messages(self) -> str: - return json.dumps( - [{"role": "system", "content": self.system_msg}] + self.messages, indent=2 - ) - - -class OpenAIAgent( - Agent[ChatCompletion, Literal["gpt-3.5", "gpt-4", "gpt-4o", "gpt-4o-mini"]] -): - - def _add_system_message(self, message: str): - self.messages.append({"role": "system", "content": message}) - - async def add_message(self, message: str) -> ChatCompletion: - self.messages.append({"role": "user", "content": message}) - - spec = { - "messages": self.messages, - "model": self.model, - "temperature": self.temperature, - "max_tokens": self.max_tokens, - } - - res = await async_openai_request(spec, self.api_config) - self.messages.append( - {"role": "assistant", "content": res.choices[0].message.content} - ) - return res - - def dump_messages(self) -> str: - return json.dumps(self.messages, indent=2) - - -class GoogleAgent(Agent[AsyncGenerateContentResponse, str]): - - def _add_system_message(self, message: str): - self.system_message = message - - async def add_message(self, message: str) -> AsyncGenerateContentResponse: - self.messages.append({"role": "user", "parts": message}) - - res = await async_gemini_request( - system_message=self.system_message, - llm_config_dto=self.api_config, - model_name="gemini-1.5-flash", - contents=self.messages, - ) - - res.candidates[0].content.parts[0] - self.messages.append( - {"role": "assistant", "parts": res.candidates[0].content.parts} - ) - - return res - - def dump_messages(self) -> str: - return json.dumps( - [{"role": "system", "parts": "system_message"}] + self.messages, indent=2 - ) \ No newline at end of file diff --git a/dendrite/logic/local/get_element/agents/prompts/__init__.py b/dendrite/logic/local/get_element/agents/prompts/__init__.py deleted file mode 100644 index b91825e..0000000 --- a/dendrite/logic/local/get_element/agents/prompts/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -def load_prompt(prompt_path: str) -> str: - with open(prompt_path, "r") as f: - prompt = f.read() - return prompt - - -SEGMENT_PROMPT = load_prompt( - "dendrite_server_merge/core/web_scraping_agent/get_interactable/agents/prompts/segment.prompt" -) -SELECT_PROMPT = load_prompt( - "dendrite_server_merge/core/web_scraping_agent/get_interactable/agents/prompts/select.prompt" -) diff --git a/dendrite/logic/local/get_element/cached_selector.py b/dendrite/logic/local/get_element/cached_selector.py deleted file mode 100644 index beab8ce..0000000 --- a/dendrite/logic/local/get_element/cached_selector.py +++ /dev/null @@ -1,56 +0,0 @@ -from datetime import datetime -from typing import List, Optional -from urllib.parse import urlparse -from bs4 import BeautifulSoup -from loguru import logger -from pydantic import BaseModel - -from dendrite.logic.interfaces.cache import CacheProtocol - - -class Selector(BaseModel): - selector: str - prompt: str - url: str - netloc: str - created_at: str - - -def deserialize_selector(selector_from_db) -> Selector: - return Selector( - selector=str(selector_from_db["selector"]), - prompt=selector_from_db.get("prompts", ""), - url=selector_from_db.get("url", ""), - netloc=selector_from_db.get("netloc", ""), - created_at=selector_from_db.get("created_at", ""), - ) - - -async def get_selector_from_db( - url: str, prompt: str, cache: CacheProtocol[Selector] -) -> Optional[Selector]: - - netloc = urlparse(url).netloc - - return cache.get({"netloc": netloc, "prompt": prompt}) - -async def add_selector_in_db( - prompt: str, bs4_selector: str, url: str -): - - created_at = datetime.now().isoformat() - netloc = urlparse(url).netloc - selector: Selector = Selector( - prompt=prompt, - selector=bs4_selector, - url=url, - netloc=netloc, - created_at=created_at, - ) - serialized = selector.model_dump() - # res = await selector_collection.insert_one(serialized) - # return str(res.inserted_id) - - - - diff --git a/dendrite/logic/local/get_element/main.py b/dendrite/logic/local/get_element/main.py deleted file mode 100644 index 650b2ef..0000000 --- a/dendrite/logic/local/get_element/main.py +++ /dev/null @@ -1,101 +0,0 @@ - -from typing import Optional -from anthropic import BaseModel -from bs4 import BeautifulSoup, Tag -from loguru import logger -from dendrite.browser.async_api._core.models.api_config import APIConfig -from dendrite.browser.async_api._core.models.page_information import PageInformation -from dendrite.browser.sync_api._api.response.get_element_response import GetElementResponse -from dendrite.logic.interfaces.cache import CacheProtocol -from dendrite.logic.local.dom.css import check_if_selector_successful, find_css_selector -from .hanifi_search import hanifi_search -from dendrite.logic.local.get_element.cached_selector import add_selector_in_db, get_selector_from_db -from dendrite.logic.local.get_element.dom import remove_hidden_elements - - -class GetElementDTO(BaseModel): - page_information: PageInformation - prompt: str - api_config: APIConfig - use_cache: bool = True - only_one: bool - force_use_cache: bool = False - - -async def process_single_prompt( - get_elements_dto: GetElementDTO, prompt: str, user_id: str, cache: Optional[CacheProtocol] = None -) -> GetElementResponse: - - soup = BeautifulSoup(get_elements_dto.page_information.raw_html, "lxml") - - if get_elements_dto.use_cache and cache: - res = await check_cache(soup, get_elements_dto.page_information.url, prompt, get_elements_dto.only_one, cache) - if res: - return res - - - if get_elements_dto.force_use_cache: - return GetElementResponse( - selectors=[], - status="failed", - message="Forced to use cache, but no cached selectors found", - used_cache=False, - ) - - soup_without_hidden_elements = remove_hidden_elements(soup) - - if get_elements_dto.only_one: - interactables_res = await hanifi_search( - soup_without_hidden_elements, - prompt, - get_elements_dto.api_config, - get_elements_dto.page_information.time_since_frame_navigated, - ) - interactable = interactables_res[0] - - if interactable.status == "success": - tag = soup.find(attrs={"d-id": interactable.dendrite_id}) - if isinstance(tag, Tag): - selector = find_css_selector(tag, soup) - await add_selector_in_db( - prompt, - bs4_selector=selector, - url=get_elements_dto.page_information.url, - ) - return GetElementResponse( - selectors=[selector], - message=interactable.reason, - status="success", - used_cache=False, - ) - else: - return GetElementResponse( - message=interactable.reason, - status=interactable.status, - used_cache=False, - ) - - -async def get_element_selector_action( - get_elements_dto: GetElementDTO, user_id: str -) -> GetElementResponse: - return await process_single_prompt( - get_elements_dto, get_elements_dto.prompt, user_id - ) - -async def check_cache(soup: BeautifulSoup, url: str, prompt: str, only_one: bool, cache: CacheProtocol) -> Optional[GetElementResponse]: - db_selectors = await get_selector_from_db( - url, prompt, cache - ) - - if db_selectors is None: - return None - - successful_selectors = [] - - if check_if_selector_successful(db_selectors.selector, soup, only_one): - return GetElementResponse( - selectors=successful_selectors, - status="success", - used_cache=True, - ) diff --git a/dendrite/logic/verify_interaction.py b/dendrite/logic/verify_interaction.py new file mode 100644 index 0000000..a4b269e --- /dev/null +++ b/dendrite/logic/verify_interaction.py @@ -0,0 +1,94 @@ +import json +from typing import List + +from bs4 import BeautifulSoup + +from dendrite.logic.llm.agent import LLM, Agent, Message +from dendrite.logic.llm.config import llm_config +from dendrite.models.dto.make_interaction_dto import VerifyActionDTO +from dendrite.models.response.interaction_response import InteractionResponse + + +async def verify_action( + make_interaction_dto: VerifyActionDTO, +) -> InteractionResponse: + + if ( + make_interaction_dto.interaction_type == "fill" + and make_interaction_dto.value == "" + ): + raise Exception(f"Error: You need to specify the keys you want to send.") + + interaction_verb = "" + if make_interaction_dto.interaction_type == "click": + interaction_verb = "clicked on" + elif make_interaction_dto.interaction_type == "fill": + interaction_verb = "sent keys to" + + + locator_desc = "" + if make_interaction_dto.dendrite_id != "": + locator_desc = "the dendrite id '{element_dendrite_id}'" + + expected_outcome = ( + "" + if make_interaction_dto.expected_outcome == None + else f"The expected outcome is: '{make_interaction_dto.expected_outcome}'" + ) + prompt = f"I {interaction_verb} a <{make_interaction_dto.tag_name}> element with {locator_desc}. {expected_outcome}" + + messages: List[Message] =[ + { + "role": "user", + "content": [], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt, + }, + { + "type": "text", + "text": "Here is the viewport before the interaction:", + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{make_interaction_dto.screenshot_before}" + }, + }, + { + "type": "text", + "text": "Here is the viewport after the interaction:", + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{make_interaction_dto.screenshot_after}" + }, + }, + { + "type": "text", + "text": """Based of the expected outcome, please output a json object that either confirms that the interaction was successful or that it failed. Output a json object like this with no description or backticks, just valid json. {"status": "success" | "failed", "message": "Give a short description of what happened and if the interaction completed successfully or failed to reach the expected outcome, write max 100 characters."}""", + }, + ], + }, + ] + + + default = LLM(model="gpt-4o", max_tokens=150) + llm = Agent(llm_config.get("verify_action", default)) + + res = await llm.call_llm(messages) + try: + dict_res = json.loads(res) + return InteractionResponse( + message=dict_res["message"], + status=dict_res["status"], + ) + except: + pass + + raise Exception("Failed to parse interaction page delta.") diff --git a/dendrite/models/api_config.py b/dendrite/models/api_config.py index c78502c..ad58987 100644 --- a/dendrite/models/api_config.py +++ b/dendrite/models/api_config.py @@ -1,7 +1,6 @@ from typing import Optional -from anthropic import BaseModel -from pydantic import model_validator +from pydantic import BaseModel, model_validator from dendrite.browser._common._exceptions.dendrite_exception import MissingApiKeyError @@ -30,4 +29,4 @@ def _check_api_keys(cls, values): raise MissingApiKeyError( "A valid dendrite_api_key must be provided. Make sure you have set the DENDRITE_API_KEY environment variable or passed it to the Dendrite." ) - return values \ No newline at end of file + return values diff --git a/dendrite/models/dto/ask_page_dto.py b/dendrite/models/dto/ask_page_dto.py index 33a4921..ad039de 100644 --- a/dendrite/models/dto/ask_page_dto.py +++ b/dendrite/models/dto/ask_page_dto.py @@ -1,12 +1,11 @@ from typing import Any, Optional + from pydantic import BaseModel from dendrite.models.page_information import PageInformation - class AskPageDTO(BaseModel): prompt: str return_schema: Optional[Any] page_information: PageInformation - diff --git a/dendrite/models/dto/extract_dto.py b/dendrite/models/dto/extract_dto.py index 50b465a..dda4baa 100644 --- a/dendrite/models/dto/extract_dto.py +++ b/dendrite/models/dto/extract_dto.py @@ -1,14 +1,13 @@ import json -from typing import Any, List, Optional +from typing import Any, Optional + from pydantic import BaseModel -from dendrite.browser.async_api._core.models.api_config import APIConfig from dendrite.models.page_information import PageInformation class ExtractDTO(BaseModel): page_information: PageInformation - api_config: APIConfig prompt: str return_data_json_schema: Any @@ -25,3 +24,21 @@ def combined_prompt(self) -> str: else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" ) return f"Task: {self.prompt}{json_schema_prompt}" + + +class TryRunScriptDTO(BaseModel): + url: str + raw_html: str + prompt: str + db_prompt: Optional[str] = None + return_data_json_schema: Any + + + @property + def combined_prompt(self) -> str: + json_schema_prompt = ( + "" + if self.return_data_json_schema == None + else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" + ) + return f"Task: {self.prompt}{json_schema_prompt}" \ No newline at end of file diff --git a/dendrite/models/dto/get_elements_dto.py b/dendrite/models/dto/get_elements_dto.py index e1d9bf8..95bb126 100644 --- a/dendrite/models/dto/get_elements_dto.py +++ b/dendrite/models/dto/get_elements_dto.py @@ -1,10 +1,10 @@ from typing import Dict, Union + from pydantic import BaseModel from dendrite.models.page_information import PageInformation - class CheckSelectorCacheDTO(BaseModel): url: str prompt: Union[str, Dict[str, str]] diff --git a/dendrite/models/dto/make_interaction_dto.py b/dendrite/models/dto/make_interaction_dto.py new file mode 100644 index 0000000..db0731b --- /dev/null +++ b/dendrite/models/dto/make_interaction_dto.py @@ -0,0 +1,19 @@ +from typing import Literal, Optional + +from pydantic import BaseModel + +from dendrite.models.page_information import PageDiffInformation + +InteractionType = Literal["click", "fill", "hover"] + + +class VerifyActionDTO(BaseModel): + url: str + dendrite_id: str + interaction_type: InteractionType + tag_name: str + value: Optional[str] = None + expected_outcome: str + screenshot_before: str + screenshot_after: str + diff --git a/dendrite/models/page_information.py b/dendrite/models/page_information.py index 8d0dae2..aded4d5 100644 --- a/dendrite/models/page_information.py +++ b/dendrite/models/page_information.py @@ -1,4 +1,4 @@ -from anthropic import BaseModel +from pydantic import BaseModel class PageInformation(BaseModel): @@ -6,3 +6,10 @@ class PageInformation(BaseModel): raw_html: str screenshot_base64: str time_since_frame_navigated: float + + +class PageDiffInformation(BaseModel): + screenshot_before: str + screenshot_after: str + page_before: PageInformation + page_after: PageInformation diff --git a/dendrite/logic/hosted/_api/response/ask_page_response.py b/dendrite/models/response/ask_page_response.py similarity index 100% rename from dendrite/logic/hosted/_api/response/ask_page_response.py rename to dendrite/models/response/ask_page_response.py diff --git a/dendrite/models/response/extract_page_response.py b/dendrite/models/response/extract_page_response.py deleted file mode 100644 index c714404..0000000 --- a/dendrite/models/response/extract_page_response.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import Any, Generic, List, Optional, TypeVar -from pydantic import BaseModel -from dendrite.browser._common.types import Status - - -T = TypeVar("T") - - -class ExtractPageResponse(BaseModel, Generic[T]): - return_data: T - message: str - created_script: Optional[str] = None - status: Status - used_cache: bool diff --git a/dendrite/logic/hosted/_api/response/extract_response.py b/dendrite/models/response/extract_response.py similarity index 65% rename from dendrite/logic/hosted/_api/response/extract_response.py rename to dendrite/models/response/extract_response.py index dd7f6d3..1d0035b 100644 --- a/dendrite/logic/hosted/_api/response/extract_response.py +++ b/dendrite/models/response/extract_response.py @@ -1,15 +1,16 @@ from typing import Generic, Optional, TypeVar -from pydantic import BaseModel -from dendrite.browser.async_api._common.status import Status +from pydantic import BaseModel +from dendrite.browser._common.types import Status T = TypeVar("T") - class ExtractResponse(BaseModel, Generic[T]): - return_data: T + status: Status message: str + return_data: Optional[T] = None + used_cache: bool = False created_script: Optional[str] = None - status: Status - used_cache: bool + + diff --git a/dendrite/logic/hosted/_api/response/get_element_response.py b/dendrite/models/response/get_element_response.py similarity index 77% rename from dendrite/logic/hosted/_api/response/get_element_response.py rename to dendrite/models/response/get_element_response.py index 8fb8fa5..2c8e07a 100644 --- a/dendrite/logic/hosted/_api/response/get_element_response.py +++ b/dendrite/models/response/get_element_response.py @@ -2,11 +2,14 @@ from pydantic import BaseModel -from dendrite.browser.async_api._common.status import Status +from dendrite.models.status import Status class GetElementResponse(BaseModel): status: Status + d_id: Optional[str] = None selectors: Optional[Union[List[str], Dict[str, List[str]]]] = None message: str = "" used_cache: bool = False + + diff --git a/dendrite/browser/sync_api/_api/response/interaction_response.py b/dendrite/models/response/interaction_response.py similarity index 64% rename from dendrite/browser/sync_api/_api/response/interaction_response.py rename to dendrite/models/response/interaction_response.py index 6d85f96..6bd1879 100644 --- a/dendrite/browser/sync_api/_api/response/interaction_response.py +++ b/dendrite/models/response/interaction_response.py @@ -1,5 +1,6 @@ from pydantic import BaseModel -from dendrite.browser.sync_api._common.status import Status + +from dendrite.models.status import Status class InteractionResponse(BaseModel): diff --git a/dendrite/models/scripts.py b/dendrite/models/scripts.py new file mode 100644 index 0000000..3d8ee61 --- /dev/null +++ b/dendrite/models/scripts.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel + +class Script(BaseModel): + url: str + domain: str + script: str + created_at: str diff --git a/dendrite/models/selector.py b/dendrite/models/selector.py new file mode 100644 index 0000000..a84feb1 --- /dev/null +++ b/dendrite/models/selector.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel + +class Selector(BaseModel): + selector: str + prompt: str + url: str + netloc: str + created_at: str diff --git a/dendrite/models/status.py b/dendrite/models/status.py new file mode 100644 index 0000000..8d5beac --- /dev/null +++ b/dendrite/models/status.py @@ -0,0 +1,4 @@ +from typing import Literal + + +Status = Literal["success", "failed", "loading", "impossible"] \ No newline at end of file diff --git a/scripts/generate_sync.py b/scripts/generate_sync.py index b28b2df..8bdee38 100644 --- a/scripts/generate_sync.py +++ b/scripts/generate_sync.py @@ -1,10 +1,10 @@ -import os import ast -import shutil import logging +import os +import shutil import subprocess import sys -from typing import Dict, Any +from typing import Any, Dict logging.basicConfig(level=logging.WARNING) From 4d4e3e44ead2b9242194559b5c166c96e6fd9687 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Fri, 29 Nov 2024 16:57:20 +0100 Subject: [PATCH 04/18] small bug fixes after refactor --- dendrite/__init__.py | 39 +++++++------------ dendrite/browser/async_api/_core/_utils.py | 11 ++++-- .../browser/async_api/_core/mixin/extract.py | 2 +- dendrite/logic/ask/ask.py | 30 +++++++------- dendrite/logic/cache/element_cache.py | 2 +- dendrite/logic/code/code_session.py | 3 +- dendrite/logic/extract/extract_agent.py | 15 ++++--- dendrite/logic/extract/scroll_agent.py | 27 +++++++------ dendrite/logic/get_element/get_element.py | 13 +++---- 9 files changed, 67 insertions(+), 75 deletions(-) diff --git a/dendrite/__init__.py b/dendrite/__init__.py index 310bd90..9d84deb 100644 --- a/dendrite/__init__.py +++ b/dendrite/__init__.py @@ -1,18 +1,11 @@ -# import sys -# from loguru import logger -# from dendrite.browser.async_api import ( -# AsyncDendrite, -# AsyncElement, -# AsyncPage, -# AsyncElementsResponse, -# ) - -# from dendrite.browser.sync_api import ( -# Dendrite, -# Element, -# Page, -# ElementsResponse, -# ) +import sys +from loguru import logger +from dendrite.browser.async_api import ( + AsyncDendrite, + AsyncElement, + AsyncPage, + AsyncElementsResponse, +) # logger.remove() @@ -21,13 +14,9 @@ # logger.add(sys.stderr, level="INFO", format=fmt) -# __all__ = [ -# "AsyncDendrite", -# "AsyncElement", -# "AsyncPage", -# "AsyncElementsResponse", -# "Dendrite", -# "Element", -# "Page", -# "ElementsResponse", -# ] +__all__ = [ + "AsyncDendrite", + "AsyncElement", + "AsyncPage", + "AsyncElementsResponse", +] diff --git a/dendrite/browser/async_api/_core/_utils.py b/dendrite/browser/async_api/_core/_utils.py index 4e91bf1..9b01b8d 100644 --- a/dendrite/browser/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_core/_utils.py @@ -38,14 +38,17 @@ async def get_iframe_path(frame: Frame): continue # Skip the main frame try: iframe_element = await frame.frame_element() + + iframe_id = await iframe_element.get_attribute("d-id") + if iframe_id is None: + continue + iframe_path = await get_iframe_path(frame) except Error as e: continue - iframe_id = await iframe_element.get_attribute("d-id") - if iframe_id is None: - continue - iframe_path = await get_iframe_path(frame) + if iframe_path is None: continue + try: await frame.evaluate( GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, {"frame_path": iframe_path} diff --git a/dendrite/browser/async_api/_core/mixin/extract.py b/dendrite/browser/async_api/_core/mixin/extract.py index f72dfae..68c593f 100644 --- a/dendrite/browser/async_api/_core/mixin/extract.py +++ b/dendrite/browser/async_api/_core/mixin/extract.py @@ -203,7 +203,7 @@ async def attempt_extraction_with_backoff( if res.status == "success": logger.success( - f"Extraction successful: '{res.message}'\nUsed cache: {res.used_cache}\nUsed script:\n\n{res.created_script}" + f"Extraction successful: '{res.message}'\nUsed cache: {res.used_cache}" ) return res diff --git a/dendrite/logic/ask/ask.py b/dendrite/logic/ask/ask.py index 272ee1b..c210afc 100644 --- a/dendrite/logic/ask/ask.py +++ b/dendrite/logic/ask/ask.py @@ -4,6 +4,8 @@ import json_repair from jsonschema import validate +from openai.types.chat.chat_completion_content_part_param import ChatCompletionContentPartParam + from dendrite.logic.llm.agent import Agent, Message from dendrite.logic.llm.config import llm_config @@ -115,7 +117,7 @@ async def ask_page_action( def generate_ask_page_prompt( ask_page_dto: AskPageDTO, image_segments: list, scrolled_to_segment_i: int = 0 -) -> list: +) -> List[ChatCompletionContentPartParam]: # Generate scroll down hint based on number of segments scroll_down_hint = ( "" @@ -143,7 +145,7 @@ def generate_ask_page_prompt( ) # Construct the main prompt content - content = [ + content: List[ChatCompletionContentPartParam]= [ { "type": "text", "text": f"""Please look at the page and return data that matches the requested schema and prompt. @@ -184,20 +186,17 @@ def generate_ask_page_prompt( Here is a screenshot of the viewport:""", }, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": image_segments[scrolled_to_segment_i], - }, + {"type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_segments[scrolled_to_segment_i]}" + } }, ] return content -def generate_scroll_prompt(image_segments: list, next_segment: int) -> list: +def generate_scroll_prompt(image_segments: list, next_segment: int) -> List[ChatCompletionContentPartParam]: """ Generates the prompt for scrolling to next segment. @@ -219,13 +218,10 @@ def generate_scroll_prompt(image_segments: list, next_segment: int) -> list: "type": "text", "text": f"""You have scrolled down. You are viewing segment {next_segment+1}/{len(image_segments)}.{last_segment_reminder} Here is the new viewport:""", }, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": image_segments[next_segment], - }, + {"type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_segments[next_segment]}" + } }, ] diff --git a/dendrite/logic/cache/element_cache.py b/dendrite/logic/cache/element_cache.py index 650ea59..87caab7 100644 --- a/dendrite/logic/cache/element_cache.py +++ b/dendrite/logic/cache/element_cache.py @@ -5,4 +5,4 @@ from dendrite.models.scripts import Script from dendrite.models.selector import Selector -element_cache = FileCache(Selector, "./cache/get_element.json") +element_cache = FileCache(Selector, "./cache/get_element.json") \ No newline at end of file diff --git a/dendrite/logic/code/code_session.py b/dendrite/logic/code/code_session.py index 6d93d11..6d2b4d4 100644 --- a/dendrite/logic/code/code_session.py +++ b/dendrite/logic/code/code_session.py @@ -118,9 +118,10 @@ def llm_readable_exec_res( response += "Newly created variables:" for var_name, var_value in variables.items(): show_length = 600 if var_name == "response_data" else 300 + try: # Convert var_value to string, handling potential errors - str_value = str(var_value) + str_value = str(var_value) if var_value is not None else "None" except Exception as e: logger.error(f"Error converting to string for display: {e}") str_value = f"" diff --git a/dendrite/logic/extract/extract_agent.py b/dendrite/logic/extract/extract_agent.py index 44e9660..1761833 100644 --- a/dendrite/logic/extract/extract_agent.py +++ b/dendrite/logic/extract/extract_agent.py @@ -1,7 +1,10 @@ import json import re +import sys from typing import Any, List, Optional +from loguru import logger + from dendrite.logic.cache.utils import save_script from dendrite.logic.dom.strip import mild_strip from dendrite.logic.extract.prompts import ( @@ -53,9 +56,7 @@ async def write_and_run_script( extract_page_dto.page_information.screenshot_base64, segment_height=4000 ) - scroll_agent = ScrollAgent( - self.llm_config.get("scroll_agent"), self.page_information - ) + scroll_agent = ScrollAgent(self.page_information) scroll_result = await scroll_agent.scroll_through_page( extract_page_dto.combined_prompt, image_segments=segments, @@ -114,7 +115,11 @@ def segment_large_tag(self, tag): async def code_script_from_found_expanded_html_tags( self, extract_page_dto: ExtractDTO, expanded_html, segments ): - # agent_logger.info("Starting code_script_from_found_expanded_html_tags method") + + agent_logger = logger.bind(scope="extract", step="generate_code") # agent_logger.info("Starting code_script_from_found_expanded_html_tags method") + agent_logger.remove() + fmt = "{time: HH:mm:ss.SSS} | {level: <8} | {message}" + agent_logger.add(sys.stderr, level="DEBUG", format=fmt) messages = [] user_prompt = create_script_prompt_segmented_html( @@ -141,7 +146,7 @@ async def code_script_from_found_expanded_html_tags( while iterations <= max_retries: iterations += 1 - # agent_logger.info(f"Starting iteration {iterations}") + agent_logger.debug(f"Code generation | Iteration: {iterations}") text = await self.call_llm(messages) messages.append({"role": "assistant", "content": text}) diff --git a/dendrite/logic/extract/scroll_agent.py b/dendrite/logic/extract/scroll_agent.py index 32869ba..c50d062 100644 --- a/dendrite/logic/extract/scroll_agent.py +++ b/dendrite/logic/extract/scroll_agent.py @@ -3,11 +3,13 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import List, Literal, Optional +from openai.types.chat.chat_completion_content_part_param import ChatCompletionContentPartParam from loguru import logger from dendrite.logic.llm.agent import Agent, Message from dendrite.models.page_information import PageInformation +from dendrite.logic.llm.config import llm_config ScrollActionStatus = Literal["done", "scroll_down", "loading", "error"] @@ -60,7 +62,7 @@ def parse(self, data_dict: dict, segment_i: int) -> Optional[ScrollResult]: class ScrollAgent(Agent): - def __init__(self, llm_config, page_information: PageInformation): + def __init__(self, page_information: PageInformation): super().__init__(llm_config.get("scroll_agent")) self.page_information = page_information self.choices: List[ScrollRes] = [ @@ -74,7 +76,7 @@ async def scroll_through_page( combined_prompt: str, image_segments: List[str], ) -> ScrollResult: - messages = self.create_initial_message(combined_prompt, image_segments[0]) + messages = [self.create_initial_message(combined_prompt, image_segments[0])] all_elements_to_inspect_html = [] current_segment = 0 @@ -133,8 +135,8 @@ async def process_segment(self, messages: List[Message]) -> dict: def create_initial_message( self, combined_prompt: str, first_image: str - ) -> List[Message]: - content = [ + ) -> Message: + content: List[ChatCompletionContentPartParam] = [ { "type": "text", "text": f"""You are a web scraping agent that can code scripts to solve the web scraping tasks listed below for the webpage I'll specify. Before we start coding, we need to inspect the html of the page closer. @@ -198,19 +200,16 @@ def create_initial_message( Below is a screenshot of the current page, if it looks blank or empty it could still be loading. If this is the case, don't guess what elements to inspect, respond with is loading.""", }, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": first_image, - }, + {"type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{first_image}" + } }, ] - return [ - {"role": "user", "content": content}, - ] + + msg: Message = {"role": "user", "content": content} + return msg def create_scroll_message(self, image: str) -> Message: return { diff --git a/dendrite/logic/get_element/get_element.py b/dendrite/logic/get_element/get_element.py index af694d9..d0faf88 100644 --- a/dendrite/logic/get_element/get_element.py +++ b/dendrite/logic/get_element/get_element.py @@ -2,9 +2,6 @@ from bs4 import BeautifulSoup, Tag from loguru import logger -from pydantic import BaseModel - -from dendrite.logic.cache.file_cache import FileCache from dendrite.logic.config import config from dendrite.logic.dom.css import check_if_selector_successful, find_css_selector from dendrite.logic.dom.strip import remove_hidden_elements @@ -13,9 +10,7 @@ get_selector_from_cache, ) from dendrite.models.dto.get_elements_dto import GetElementsDTO -from dendrite.models.page_information import PageInformation from dendrite.models.response.get_element_response import GetElementResponse -from dendrite.models.selector import Selector from .hanifi_search import hanifi_search @@ -23,7 +18,6 @@ async def get_element(dto: GetElementsDTO) -> GetElementResponse: - if isinstance(dto.prompt, str): return await process_prompt(dto.prompt, dto) raise ... @@ -105,8 +99,13 @@ async def check_cache( if check_if_selector_successful(db_selectors.selector, soup, only_one): return GetElementResponse( - selectors=successful_selectors, + selectors=[db_selectors.selector], status="success", used_cache=True, ) + +# async def get_cached_selector(dto: GetCachedSelectorDTO) -> Optional[Selector]: +# cache = config.element_cache +# db_selectors = await get_selector_from_cache(dto.url, dto.prompt, cache) +# return db_selectors \ No newline at end of file From 0e0519cc1ce387717c693fc01b6c56b7cfc4ffd7 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Fri, 29 Nov 2024 16:58:14 +0100 Subject: [PATCH 05/18] format with black --- .../_common/_exceptions/dendrite_exception.py | 1 - dendrite/browser/async_api/_core/_utils.py | 8 +- .../async_api/_core/dendrite_browser.py | 6 +- .../async_api/_core/dendrite_element.py | 14 +-- .../browser/async_api/_core/mixin/__init__.py | 2 - .../browser/async_api/_core/mixin/extract.py | 2 - .../async_api/_core/mixin/get_element.py | 4 - dendrite/logic/ask/ask.py | 31 ++++--- dendrite/logic/cache/element_cache.py | 2 +- dendrite/logic/cache/file_cache.py | 6 +- dendrite/logic/cache/utils.py | 5 +- dendrite/logic/config.py | 4 +- dendrite/logic/dom/strip.py | 3 +- dendrite/logic/extract/cached_script.py | 4 +- dendrite/logic/extract/compress_html.py | 1 - dendrite/logic/extract/extract.py | 45 +++++----- dendrite/logic/extract/extract_agent.py | 12 +-- dendrite/logic/extract/scroll_agent.py | 17 ++-- dendrite/logic/factory.py | 3 +- .../get_element/agents/prompts/__init__.py | 8 +- dendrite/logic/get_element/cached_selector.py | 2 +- dendrite/logic/get_element/get_element.py | 24 ++--- dendrite/logic/get_element/hanifi_search.py | 12 +-- dendrite/logic/hosted/_api/_http_client.py | 4 +- .../logic/hosted/_api/browser_api_client.py | 4 +- dendrite/logic/interfaces/async_api.py | 15 ++-- dendrite/logic/interfaces/sync_api.py | 12 ++- dendrite/logic/llm/agent.py | 4 +- dendrite/logic/llm/config.py | 32 +++++-- dendrite/logic/verify_interaction.py | 88 +++++++++---------- dendrite/models/dto/extract_dto.py | 3 +- dendrite/models/dto/make_interaction_dto.py | 1 - dendrite/models/response/extract_response.py | 3 +- .../models/response/get_element_response.py | 2 - dendrite/models/scripts.py | 1 + dendrite/models/selector.py | 1 + dendrite/models/status.py | 2 +- 37 files changed, 189 insertions(+), 199 deletions(-) diff --git a/dendrite/browser/_common/_exceptions/dendrite_exception.py b/dendrite/browser/_common/_exceptions/dendrite_exception.py index 1b2b86b..ddfdeed 100644 --- a/dendrite/browser/_common/_exceptions/dendrite_exception.py +++ b/dendrite/browser/_common/_exceptions/dendrite_exception.py @@ -111,7 +111,6 @@ class IncorrectOutcomeError(BaseDendriteException): """ - class BrowserNotLaunchedError(BaseDendriteException): """ Exception raised when the browser is not launched. diff --git a/dendrite/browser/async_api/_core/_utils.py b/dendrite/browser/async_api/_core/_utils.py index 9b01b8d..62fd9f6 100644 --- a/dendrite/browser/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_core/_utils.py @@ -35,20 +35,20 @@ async def get_iframe_path(frame: Frame): for frame in page.frames: if frame.parent_frame is None: - continue # Skip the main frame + continue # Skip the main frame try: iframe_element = await frame.frame_element() - + iframe_id = await iframe_element.get_attribute("d-id") if iframe_id is None: continue iframe_path = await get_iframe_path(frame) except Error as e: continue - + if iframe_path is None: continue - + try: await frame.evaluate( GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, {"frame_path": iframe_path} diff --git a/dendrite/browser/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/_core/dendrite_browser.py index a9706c5..4bb6e40 100644 --- a/dendrite/browser/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/_core/dendrite_browser.py @@ -39,7 +39,7 @@ MarkdownMixin, ScreenshotMixin, WaitForMixin, - ) +) from dendrite.browser.remote import Providers from dendrite.logic.interfaces.async_api import LocalProtocol, LogicAPIProtocol @@ -141,7 +141,7 @@ async def _get_page(self) -> AsyncPage: active_page = await self.get_active_page() return active_page - def _get_logic_api(self) -> 'LogicAPIProtocol': + def _get_logic_api(self) -> "LogicAPIProtocol": return self._browser_api_client def _get_dendrite_browser(self) -> "AsyncDendrite": @@ -286,8 +286,6 @@ async def _launch(self): self._playwright, self._playwright_options ) - - self.browser_context = ( browser.contexts[0] if len(browser.contexts) > 0 diff --git a/dendrite/browser/async_api/_core/dendrite_element.py b/dendrite/browser/async_api/_core/dendrite_element.py index 16dfaec..5af8fdc 100644 --- a/dendrite/browser/async_api/_core/dendrite_element.py +++ b/dendrite/browser/async_api/_core/dendrite_element.py @@ -55,7 +55,7 @@ async def wrapper( page_before = await self._dendrite_browser.get_active_page() page_before_info = await page_before.get_page_information() - soup = await page_before._get_previous_soup() + soup = await page_before._get_previous_soup() screenshot_before = page_before_info.screenshot_base64 tag_name = soup.find(attrs={"d-id": self.dendrite_id}) # Call the original method here @@ -65,12 +65,13 @@ async def wrapper( *args, **kwargs, ) - + await self._wait_for_page_changes(page_before.url) page_after = await self._dendrite_browser.get_active_page() - screenshot_after = await page_after.screenshot_manager.take_full_page_screenshot() - + screenshot_after = ( + await page_after.screenshot_manager.take_full_page_screenshot() + ) dto = VerifyActionDTO( url=page_before.url, @@ -79,14 +80,13 @@ async def wrapper( expected_outcome=expected_outcome, screenshot_before=screenshot_before, screenshot_after=screenshot_after, - tag_name = str(tag_name), + tag_name=str(tag_name), ) res = await self._browser_api_client.verify_action(dto) if res.status == "failed": raise IncorrectOutcomeError( - message=res.message, - screenshot_base64=screenshot_after + message=res.message, screenshot_base64=screenshot_after ) return res diff --git a/dendrite/browser/async_api/_core/mixin/__init__.py b/dendrite/browser/async_api/_core/mixin/__init__.py index 140ba4a..fd6b7cd 100644 --- a/dendrite/browser/async_api/_core/mixin/__init__.py +++ b/dendrite/browser/async_api/_core/mixin/__init__.py @@ -1,4 +1,3 @@ - from .ask import AskMixin from .click import ClickMixin from .extract import ExtractionMixin @@ -21,4 +20,3 @@ "ScreenshotMixin", "WaitForMixin", ] - diff --git a/dendrite/browser/async_api/_core/mixin/extract.py b/dendrite/browser/async_api/_core/mixin/extract.py index 68c593f..0d1d5a1 100644 --- a/dendrite/browser/async_api/_core/mixin/extract.py +++ b/dendrite/browser/async_api/_core/mixin/extract.py @@ -162,8 +162,6 @@ async def extract( return None - - async def attempt_extraction_with_backoff( obj: DendritePageProtocol, prompt: str, diff --git a/dendrite/browser/async_api/_core/mixin/get_element.py b/dendrite/browser/async_api/_core/mixin/get_element.py index 3ba39eb..bb171db 100644 --- a/dendrite/browser/async_api/_core/mixin/get_element.py +++ b/dendrite/browser/async_api/_core/mixin/get_element.py @@ -196,13 +196,11 @@ async def _get_element( Union[AsyncElement, List[AsyncElement], AsyncElementsResponse]: The retrieved element, list of elements, or response object. """ - start_time = time.time() # First, let's check if there is a cached selector page = await self._get_page() - # If we have cached elements, attempt to use them with an exponentation backoff if use_cache == True: logger.debug("Attempting to use cached selectors") @@ -240,8 +238,6 @@ async def _get_element( return None - - async def attempt_with_backoff( obj: DendritePageProtocol, prompt_or_elements: Union[str, Dict[str, str]], diff --git a/dendrite/logic/ask/ask.py b/dendrite/logic/ask/ask.py index c210afc..9f9ecff 100644 --- a/dendrite/logic/ask/ask.py +++ b/dendrite/logic/ask/ask.py @@ -4,7 +4,9 @@ import json_repair from jsonschema import validate -from openai.types.chat.chat_completion_content_part_param import ChatCompletionContentPartParam +from openai.types.chat.chat_completion_content_part_param import ( + ChatCompletionContentPartParam, +) from dendrite.logic.llm.agent import Agent, Message @@ -34,12 +36,13 @@ async def ask_page_action( while iteration < max_iterations: iteration += 1 - text = await agent.call_llm(messages) - messages.append({ - "role": "assistant", - "content": text, - }) + messages.append( + { + "role": "assistant", + "content": text, + } + ) json_pattern = r"```json(.*?)```" @@ -145,7 +148,7 @@ def generate_ask_page_prompt( ) # Construct the main prompt content - content: List[ChatCompletionContentPartParam]= [ + content: List[ChatCompletionContentPartParam] = [ { "type": "text", "text": f"""Please look at the page and return data that matches the requested schema and prompt. @@ -186,17 +189,20 @@ def generate_ask_page_prompt( Here is a screenshot of the viewport:""", }, - {"type": "image_url", + { + "type": "image_url", "image_url": { "url": f"data:image/jpeg;base64,{image_segments[scrolled_to_segment_i]}" - } + }, }, ] return content -def generate_scroll_prompt(image_segments: list, next_segment: int) -> List[ChatCompletionContentPartParam]: +def generate_scroll_prompt( + image_segments: list, next_segment: int +) -> List[ChatCompletionContentPartParam]: """ Generates the prompt for scrolling to next segment. @@ -218,10 +224,11 @@ def generate_scroll_prompt(image_segments: list, next_segment: int) -> List[Chat "type": "text", "text": f"""You have scrolled down. You are viewing segment {next_segment+1}/{len(image_segments)}.{last_segment_reminder} Here is the new viewport:""", }, - {"type": "image_url", + { + "type": "image_url", "image_url": { "url": f"data:image/jpeg;base64,{image_segments[next_segment]}" - } + }, }, ] diff --git a/dendrite/logic/cache/element_cache.py b/dendrite/logic/cache/element_cache.py index 87caab7..05e8106 100644 --- a/dendrite/logic/cache/element_cache.py +++ b/dendrite/logic/cache/element_cache.py @@ -5,4 +5,4 @@ from dendrite.models.scripts import Script from dendrite.models.selector import Selector -element_cache = FileCache(Selector, "./cache/get_element.json") \ No newline at end of file +element_cache = FileCache(Selector, "./cache/get_element.json") diff --git a/dendrite/logic/cache/file_cache.py b/dendrite/logic/cache/file_cache.py index 8b9afd7..12405d1 100644 --- a/dendrite/logic/cache/file_cache.py +++ b/dendrite/logic/cache/file_cache.py @@ -10,7 +10,9 @@ class FileCache(Generic[T]): - def __init__(self, model_class: Type[T], filepath: Union[str, Path] = "./cache.json"): + def __init__( + self, model_class: Type[T], filepath: Union[str, Path] = "./cache.json" + ): self.filepath = Path(filepath) self.model_class = model_class self.lock = threading.RLock() @@ -47,7 +49,7 @@ def _save_cache(self, cache_dict: Dict[str, T]) -> None: } self.filepath.write_text(json.dumps(serializable_dict, indent=2)) - def get(self, key: Union[str,Dict[str,str]]) -> Union[T, None]: + def get(self, key: Union[str, Dict[str, str]]) -> Union[T, None]: hashed_key = self.hash(key) return self.cache.get(hashed_key) diff --git a/dendrite/logic/cache/utils.py b/dendrite/logic/cache/utils.py index dc78418..a3243b5 100644 --- a/dendrite/logic/cache/utils.py +++ b/dendrite/logic/cache/utils.py @@ -1,5 +1,3 @@ - - from datetime import datetime from typing import Optional from urllib.parse import urlparse @@ -15,5 +13,6 @@ def save_script(code: str, prompt: str, url: str): ) extract_cache.ExtractCache.set({"prompt": prompt, "domain": domain}, script) + def get_script(prompt: str, domain: str) -> Optional[Script]: - return extract_cache.ExtractCache.get({"prompt": prompt, "domain": domain}) \ No newline at end of file + return extract_cache.ExtractCache.get({"prompt": prompt, "domain": domain}) diff --git a/dendrite/logic/config.py b/dendrite/logic/config.py index 028da35..e2f88a9 100644 --- a/dendrite/logic/config.py +++ b/dendrite/logic/config.py @@ -6,10 +6,10 @@ class Config: def __init__(self): - self.cache_path = Path("./cache") + self.cache_path = Path("./cache") self.llm_config = "8udjsad" self.extract_cache = FileCache(Script, self.cache_path / "extract.json") self.element_cache = FileCache(Selector, self.cache_path / "get_element.json") -config = Config() \ No newline at end of file +config = Config() diff --git a/dendrite/logic/dom/strip.py b/dendrite/logic/dom/strip.py index e10cfbc..fb4dc43 100644 --- a/dendrite/logic/dom/strip.py +++ b/dendrite/logic/dom/strip.py @@ -149,11 +149,10 @@ def strip_soup(soup: BeautifulSoup) -> BeautifulSoup: return stripped_soup - def remove_hidden_elements(soup: BeautifulSoup): # data-hidden is added by DendriteBrowser when an element is not visible new_soup = copy.copy(soup) elems = new_soup.find_all(attrs={"data-hidden": True}) for elem in elems: elem.extract() - return new_soup \ No newline at end of file + return new_soup diff --git a/dendrite/logic/extract/cached_script.py b/dendrite/logic/extract/cached_script.py index 7554d4d..7420186 100644 --- a/dendrite/logic/extract/cached_script.py +++ b/dendrite/logic/extract/cached_script.py @@ -19,7 +19,7 @@ async def get_working_cached_script( if len(url) == 0: raise Exception("Domain must be specified") - scripts: List[Script] = [get_script(prompt, domain) or ...] + scripts: List[Script] = [get_script(prompt, domain) or ...] logger.debug( f"Found {len(scripts)} scripts in cache | Prompt: {prompt} in domain: {domain}" ) @@ -39,4 +39,4 @@ async def get_working_cached_script( raise Exception( f"No working script found in cache even though {len(scripts)} scripts were available | Prompt: '{prompt}' in domain: '{domain}'" - ) \ No newline at end of file + ) diff --git a/dendrite/logic/extract/compress_html.py b/dendrite/logic/extract/compress_html.py index af09f5b..f7fb3fc 100644 --- a/dendrite/logic/extract/compress_html.py +++ b/dendrite/logic/extract/compress_html.py @@ -422,7 +422,6 @@ def is_effectively_empty(element): if len(str(self.root)) < 1500: return self.root.prettify() - # print("time: ", end_time - start_time) # remove_double_nested(self.root) diff --git a/dendrite/logic/extract/extract.py b/dendrite/logic/extract/extract.py index 957d92f..f37cfc4 100644 --- a/dendrite/logic/extract/extract.py +++ b/dendrite/logic/extract/extract.py @@ -14,16 +14,14 @@ # from your_module import WebScrapingAgent, run_script_if_cached -async def test_cache( - extract_dto: ExtractDTO -) -> Optional[ExtractResponse]: +async def test_cache(extract_dto: ExtractDTO) -> Optional[ExtractResponse]: try: cached_script_res = await get_working_cached_script( - extract_dto.combined_prompt, + extract_dto.combined_prompt, extract_dto.page_information.raw_html, extract_dto.page_information.url, - extract_dto.return_data_json_schema + extract_dto.return_data_json_schema, ) if cached_script_res is None: @@ -51,7 +49,10 @@ class InMemoryLockManager: events = {} global_lock = asyncio.Lock() - def __init__(self, extract_page_dto: ExtractDTO,): + def __init__( + self, + extract_page_dto: ExtractDTO, + ): self.key = self.generate_key(extract_page_dto) def generate_key(self, extract_page_dto: ExtractDTO) -> str: @@ -102,10 +103,7 @@ async def wait_for_notification( InMemoryLockManager.events.pop(self.key, None) - -async def extract( - extract_page_dto: ExtractDTO -) -> ExtractResponse: +async def extract(extract_page_dto: ExtractDTO) -> ExtractResponse: # Check cache usage flags if extract_page_dto.use_cache or extract_page_dto.force_use_cache: res = await test_cache(extract_page_dto) @@ -137,12 +135,12 @@ async def extract( return res - -async def generate_script(extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager) -> ExtractResponse: +async def generate_script( + extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager +) -> ExtractResponse: try: extract_agent = ExtractAgent( extract_page_dto.page_information, - ) res = await extract_agent.write_and_run_script(extract_page_dto) await lock_manager.publish("done") @@ -153,13 +151,16 @@ async def generate_script(extract_page_dto: ExtractDTO, lock_manager: InMemoryLo finally: await lock_manager.release_lock() -async def wait_for_script_generation(extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager) -> Optional[ExtractResponse]: - event = await lock_manager.subscribe() - logger.info("Waiting for script to be generated") - notification_received = await lock_manager.wait_for_notification(event) - # If script was created after waiting - if notification_received: - res = await test_cache(extract_page_dto) - if res: - return res +async def wait_for_script_generation( + extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager +) -> Optional[ExtractResponse]: + event = await lock_manager.subscribe() + logger.info("Waiting for script to be generated") + notification_received = await lock_manager.wait_for_notification(event) + + # If script was created after waiting + if notification_received: + res = await test_cache(extract_page_dto) + if res: + return res diff --git a/dendrite/logic/extract/extract_agent.py b/dendrite/logic/extract/extract_agent.py index 1761833..4828d88 100644 --- a/dendrite/logic/extract/extract_agent.py +++ b/dendrite/logic/extract/extract_agent.py @@ -28,7 +28,6 @@ from ..llm.config import llm_config - class ExtractAgent(Agent): def __init__( self, @@ -41,7 +40,6 @@ def __init__( self.generated_script: Optional[str] = None self.llm_config = llm_config - def get_generated_script(self): return self.generated_script @@ -92,8 +90,8 @@ async def write_and_run_script( return await self.code_script_from_found_expanded_html_tags( extract_page_dto, expanded_html, segments ) - - raise Exception("Failed to extract data from the page") # TODO: skriv bättre + + raise Exception("Failed to extract data from the page") # TODO: skriv bättre def segment_large_tag(self, tag): segments = [] @@ -115,8 +113,10 @@ def segment_large_tag(self, tag): async def code_script_from_found_expanded_html_tags( self, extract_page_dto: ExtractDTO, expanded_html, segments ): - - agent_logger = logger.bind(scope="extract", step="generate_code") # agent_logger.info("Starting code_script_from_found_expanded_html_tags method") + + agent_logger = logger.bind( + scope="extract", step="generate_code" + ) # agent_logger.info("Starting code_script_from_found_expanded_html_tags method") agent_logger.remove() fmt = "{time: HH:mm:ss.SSS} | {level: <8} | {message}" agent_logger.add(sys.stderr, level="DEBUG", format=fmt) diff --git a/dendrite/logic/extract/scroll_agent.py b/dendrite/logic/extract/scroll_agent.py index c50d062..59fc914 100644 --- a/dendrite/logic/extract/scroll_agent.py +++ b/dendrite/logic/extract/scroll_agent.py @@ -3,7 +3,9 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import List, Literal, Optional -from openai.types.chat.chat_completion_content_part_param import ChatCompletionContentPartParam +from openai.types.chat.chat_completion_content_part_param import ( + ChatCompletionContentPartParam, +) from loguru import logger @@ -131,11 +133,8 @@ async def process_segment(self, messages: List[Message]) -> dict: logger.error(error_message) messages.append({"role": "user", "content": error_message}) raise Exception(error_message) - - def create_initial_message( - self, combined_prompt: str, first_image: str - ) -> Message: + def create_initial_message(self, combined_prompt: str, first_image: str) -> Message: content: List[ChatCompletionContentPartParam] = [ { "type": "text", @@ -200,14 +199,12 @@ def create_initial_message( Below is a screenshot of the current page, if it looks blank or empty it could still be loading. If this is the case, don't guess what elements to inspect, respond with is loading.""", }, - {"type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{first_image}" - } + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{first_image}"}, }, ] - msg: Message = {"role": "user", "content": content} return msg diff --git a/dendrite/logic/factory.py b/dendrite/logic/factory.py index c3416bf..02f2782 100644 --- a/dendrite/logic/factory.py +++ b/dendrite/logic/factory.py @@ -1,4 +1,3 @@ - # from typing import Literal, Optional # from dendrite.logic.interfaces.async_api import LogicAPIProtocol @@ -13,4 +12,4 @@ # if mode == "local": # return LocalBrowserAPI() # else: -# return BrowserAPIClient(api_config, session_id) \ No newline at end of file +# return BrowserAPIClient(api_config, session_id) diff --git a/dendrite/logic/get_element/agents/prompts/__init__.py b/dendrite/logic/get_element/agents/prompts/__init__.py index acf88e2..ef366ac 100644 --- a/dendrite/logic/get_element/agents/prompts/__init__.py +++ b/dendrite/logic/get_element/agents/prompts/__init__.py @@ -4,9 +4,5 @@ def load_prompt(prompt_path: str) -> str: return prompt -SEGMENT_PROMPT = load_prompt( - "dendrite/logic/get_element/agents/prompts/segment.prompt" -) -SELECT_PROMPT = load_prompt( - "dendrite/logic/get_element/agents/prompts/segment.prompt" -) +SEGMENT_PROMPT = load_prompt("dendrite/logic/get_element/agents/prompts/segment.prompt") +SELECT_PROMPT = load_prompt("dendrite/logic/get_element/agents/prompts/segment.prompt") diff --git a/dendrite/logic/get_element/cached_selector.py b/dendrite/logic/get_element/cached_selector.py index 9f8e782..7c57c7c 100644 --- a/dendrite/logic/get_element/cached_selector.py +++ b/dendrite/logic/get_element/cached_selector.py @@ -32,4 +32,4 @@ async def add_selector_to_cache(prompt: str, bs4_selector: str, url: str) -> Non created_at=created_at, ) - cache.set({"netloc": netloc, "prompt": prompt}, selector) \ No newline at end of file + cache.set({"netloc": netloc, "prompt": prompt}, selector) diff --git a/dendrite/logic/get_element/get_element.py b/dendrite/logic/get_element/get_element.py index d0faf88..10ba487 100644 --- a/dendrite/logic/get_element/get_element.py +++ b/dendrite/logic/get_element/get_element.py @@ -15,16 +15,13 @@ from .hanifi_search import hanifi_search - - async def get_element(dto: GetElementsDTO) -> GetElementResponse: if isinstance(dto.prompt, str): return await process_prompt(dto.prompt, dto) raise ... -async def process_prompt( - prompt: str, dto: GetElementsDTO -) -> GetElementResponse: + +async def process_prompt(prompt: str, dto: GetElementsDTO) -> GetElementResponse: soup = BeautifulSoup(dto.page_information.raw_html, "lxml") @@ -46,15 +43,18 @@ async def process_prompt( used_cache=False, ) - return await get_new_element(soup, prompt, dto ) + return await get_new_element(soup, prompt, dto) + -async def get_new_element(soup: BeautifulSoup, prompt: str, dto: GetElementsDTO) -> GetElementResponse: +async def get_new_element( + soup: BeautifulSoup, prompt: str, dto: GetElementsDTO +) -> GetElementResponse: soup_without_hidden_elements = remove_hidden_elements(soup) element = await hanifi_search( - soup_without_hidden_elements, - prompt, - dto.page_information.time_since_frame_navigated, - ) + soup_without_hidden_elements, + prompt, + dto.page_information.time_since_frame_navigated, + ) interactable = element[0] if interactable.status == "success": @@ -108,4 +108,4 @@ async def check_cache( # async def get_cached_selector(dto: GetCachedSelectorDTO) -> Optional[Selector]: # cache = config.element_cache # db_selectors = await get_selector_from_cache(dto.url, dto.prompt, cache) -# return db_selectors \ No newline at end of file +# return db_selectors diff --git a/dendrite/logic/get_element/hanifi_search.py b/dendrite/logic/get_element/hanifi_search.py index 03449d5..b3406f2 100644 --- a/dendrite/logic/get_element/hanifi_search.py +++ b/dendrite/logic/get_element/hanifi_search.py @@ -56,9 +56,7 @@ async def hanifi_search( expand_res = await get_expanded_dom(stripped_soup, prompt) if expand_res is None: - return [ - Element(status="failed", reason="No element found when expanding HTML") - ] + return [Element(status="failed", reason="No element found when expanding HTML")] expanded, tags, flat_list = expand_res @@ -92,9 +90,7 @@ async def hanifi_search( ] else: return [ - Element( - status=res.status, dendrite_id=res.d_id[0], reason=res.reason - ) + Element(status=res.status, dendrite_id=res.d_id[0], reason=res.reason) ] return [Element(status=res.status, dendrite_id=None, reason=res.reason)] @@ -108,9 +104,7 @@ async def get_relevant_tags( tasks: List[Coroutine[Any, Any, SegmentAgentReponseType]] = [] for index, segment in enumerate(segments): - tasks.append( - extract_relevant_d_ids(prompt, segment, index) - ) + tasks.append(extract_relevant_d_ids(prompt, segment, index)) results: List[SegmentAgentReponseType] = await asyncio.gather(*tasks) if results is None: diff --git a/dendrite/logic/hosted/_api/_http_client.py b/dendrite/logic/hosted/_api/_http_client.py index 55f01fc..69e0375 100644 --- a/dendrite/logic/hosted/_api/_http_client.py +++ b/dendrite/logic/hosted/_api/_http_client.py @@ -7,7 +7,6 @@ from dendrite.models.api_config import APIConfig - class HTTPClient: def __init__(self, api_config: APIConfig, session_id: Optional[str] = None): self.api_key = api_config.dendrite_api_key @@ -27,7 +26,7 @@ async def send_request( self, endpoint: str, params: Optional[dict] = None, - data: Optional[Dict[str,Any]] = None, + data: Optional[Dict[str, Any]] = None, headers: Optional[dict] = None, method: str = "GET", ) -> httpx.Response: @@ -46,7 +45,6 @@ async def send_request( # inject api_config to data if data: data["api_config"] = self.api_config.model_dump() - response = await client.request( method, url, params=params, json=data, headers=headers diff --git a/dendrite/logic/hosted/_api/browser_api_client.py b/dendrite/logic/hosted/_api/browser_api_client.py index 0b79090..d9e6220 100644 --- a/dendrite/logic/hosted/_api/browser_api_client.py +++ b/dendrite/logic/hosted/_api/browser_api_client.py @@ -12,9 +12,7 @@ class BrowserAPIClient(HTTPClient): - async def get_element( - self, dto: GetElementsDTO - ) -> GetElementResponse: + async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: res = await self.send_request( "actions/get-interaction-selector", data=dto.model_dump(), method="POST" ) diff --git a/dendrite/logic/interfaces/async_api.py b/dendrite/logic/interfaces/async_api.py index 0ebbe0f..fb36946 100644 --- a/dendrite/logic/interfaces/async_api.py +++ b/dendrite/logic/interfaces/async_api.py @@ -1,4 +1,3 @@ - from typing import Protocol from dendrite.browser.async_api._core.models.authentication import AuthSession @@ -17,11 +16,10 @@ from dendrite.logic.extract import extract from dendrite.logic import verify_interaction + class LogicAPIProtocol(Protocol): - async def get_element( - self, dto: GetElementsDTO - ) -> GetElementResponse: ... + async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: ... async def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: ... @@ -33,12 +31,15 @@ async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... class LocalProtocol(LogicAPIProtocol): async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: return await get_element.get_element(dto) - + + # async def get_cached_selectors(self, dto: CheckSelectorCacheDTO) -> GetElementResponse: + # return await get_element.get_cached_selectors(dto) + async def extract(self, dto: ExtractDTO) -> ExtractResponse: return await extract.extract(dto) - + async def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: return await verify_interaction.verify_action(dto) - + async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: return await ask.ask_page_action(dto) diff --git a/dendrite/logic/interfaces/sync_api.py b/dendrite/logic/interfaces/sync_api.py index 3eafa85..8b2f8f6 100644 --- a/dendrite/logic/interfaces/sync_api.py +++ b/dendrite/logic/interfaces/sync_api.py @@ -1,4 +1,3 @@ - import asyncio from concurrent.futures import ThreadPoolExecutor import threading @@ -48,11 +47,10 @@ def run_in_new_loop(): else: return asyncio.run_coroutine_threadsafe(coroutine, loop).result() + class LogicAPIProtocol(Protocol): - def get_element( - self, dto: GetElementsDTO - ) -> GetElementResponse: ... + def get_element(self, dto: GetElementsDTO) -> GetElementResponse: ... def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: ... @@ -64,12 +62,12 @@ def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... class LocalProtocol(LogicAPIProtocol): def get_element(self, dto: GetElementsDTO) -> GetElementResponse: return run_coroutine_sync(get_element.get_element(dto)) - + def extract(self, dto: ExtractDTO) -> ExtractResponse: return run_coroutine_sync(extract.extract(dto)) - + def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: return run_coroutine_sync(verify_interaction.verify_action(dto)) - + def ask_page(self, dto: AskPageDTO) -> AskPageResponse: return run_coroutine_sync(ask.ask_page_action(dto)) diff --git a/dendrite/logic/llm/agent.py b/dendrite/logic/llm/agent.py index 39c5298..5c179e7 100644 --- a/dendrite/logic/llm/agent.py +++ b/dendrite/logic/llm/agent.py @@ -210,13 +210,13 @@ def __init__( async def add_message(self, message: str) -> str: self.messages.append({"role": "user", "content": message}) - + text = await self.call_llm(self.messages) self.messages.append({"role": "assistant", "content": text}) return text - + async def call_llm(self, messages: List[Message]) -> str: res = await self.llm.acall(messages) diff --git a/dendrite/logic/llm/config.py b/dendrite/logic/llm/config.py index 8471716..49def1b 100644 --- a/dendrite/logic/llm/config.py +++ b/dendrite/logic/llm/config.py @@ -9,21 +9,34 @@ DEFAULT_LLM = { - "extract_agent": LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500), + "extract_agent": LLM( + "claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500 + ), "scroll_agent": LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500), - "ask_page_agent": LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500), + "ask_page_agent": LLM( + "claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500 + ), "segment_agent": LLM("gpt-4o", temperature=0, max_tokens=1500), "select_agent": LLM("claude-3-5-sonnet-20241022", temperature=0, max_tokens=1500), - "verify_action_agent": LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500), + "verify_action_agent": LLM( + "claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500 + ), } -class LLMConfig(): - def __init__(self, default_agents: Optional[Dict[str, LLM]] = None, default_llm: Optional[LLM] = None): + +class LLMConfig: + def __init__( + self, + default_agents: Optional[Dict[str, LLM]] = None, + default_llm: Optional[LLM] = None, + ): self.registered_llms: Dict[str, LLM] = DEFAULT_LLM.copy() if default_agents: self.registered_llms.update(default_agents) - self.default_llm = default_llm or LLM("claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500) + self.default_llm = default_llm or LLM( + "claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500 + ) async def register_agent(self, agent: str, llm: LLM) -> None: """ @@ -51,7 +64,12 @@ def get(self, agent: str) -> LLM: ... def get(self, agent: str, default: LLM) -> LLM: ... @overload - def get(self, agent: str, default: Optional[LLM] = ..., use_default: Literal[False] = False) -> Optional[LLM]: ... + def get( + self, + agent: str, + default: Optional[LLM] = ..., + use_default: Literal[False] = False, + ) -> Optional[LLM]: ... def get( self, diff --git a/dendrite/logic/verify_interaction.py b/dendrite/logic/verify_interaction.py index a4b269e..f697edb 100644 --- a/dendrite/logic/verify_interaction.py +++ b/dendrite/logic/verify_interaction.py @@ -25,7 +25,6 @@ async def verify_action( elif make_interaction_dto.interaction_type == "fill": interaction_verb = "sent keys to" - locator_desc = "" if make_interaction_dto.dendrite_id != "": locator_desc = "the dendrite id '{element_dendrite_id}'" @@ -37,57 +36,56 @@ async def verify_action( ) prompt = f"I {interaction_verb} a <{make_interaction_dto.tag_name}> element with {locator_desc}. {expected_outcome}" - messages: List[Message] =[ - { - "role": "user", - "content": [], - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": prompt, - }, - { - "type": "text", - "text": "Here is the viewport before the interaction:", - }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{make_interaction_dto.screenshot_before}" - }, + messages: List[Message] = [ + { + "role": "user", + "content": [], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt, + }, + { + "type": "text", + "text": "Here is the viewport before the interaction:", + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{make_interaction_dto.screenshot_before}" }, - { - "type": "text", - "text": "Here is the viewport after the interaction:", + }, + { + "type": "text", + "text": "Here is the viewport after the interaction:", + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{make_interaction_dto.screenshot_after}" }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{make_interaction_dto.screenshot_after}" - }, - }, - { - "type": "text", - "text": """Based of the expected outcome, please output a json object that either confirms that the interaction was successful or that it failed. Output a json object like this with no description or backticks, just valid json. {"status": "success" | "failed", "message": "Give a short description of what happened and if the interaction completed successfully or failed to reach the expected outcome, write max 100 characters."}""", - }, - ], - }, - ] - + }, + { + "type": "text", + "text": """Based of the expected outcome, please output a json object that either confirms that the interaction was successful or that it failed. Output a json object like this with no description or backticks, just valid json. {"status": "success" | "failed", "message": "Give a short description of what happened and if the interaction completed successfully or failed to reach the expected outcome, write max 100 characters."}""", + }, + ], + }, + ] default = LLM(model="gpt-4o", max_tokens=150) llm = Agent(llm_config.get("verify_action", default)) - + res = await llm.call_llm(messages) try: - dict_res = json.loads(res) - return InteractionResponse( - message=dict_res["message"], - status=dict_res["status"], - ) + dict_res = json.loads(res) + return InteractionResponse( + message=dict_res["message"], + status=dict_res["status"], + ) except: pass diff --git a/dendrite/models/dto/extract_dto.py b/dendrite/models/dto/extract_dto.py index dda4baa..0bb4c74 100644 --- a/dendrite/models/dto/extract_dto.py +++ b/dendrite/models/dto/extract_dto.py @@ -33,7 +33,6 @@ class TryRunScriptDTO(BaseModel): db_prompt: Optional[str] = None return_data_json_schema: Any - @property def combined_prompt(self) -> str: json_schema_prompt = ( @@ -41,4 +40,4 @@ def combined_prompt(self) -> str: if self.return_data_json_schema == None else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" ) - return f"Task: {self.prompt}{json_schema_prompt}" \ No newline at end of file + return f"Task: {self.prompt}{json_schema_prompt}" diff --git a/dendrite/models/dto/make_interaction_dto.py b/dendrite/models/dto/make_interaction_dto.py index db0731b..4fcab0f 100644 --- a/dendrite/models/dto/make_interaction_dto.py +++ b/dendrite/models/dto/make_interaction_dto.py @@ -16,4 +16,3 @@ class VerifyActionDTO(BaseModel): expected_outcome: str screenshot_before: str screenshot_after: str - diff --git a/dendrite/models/response/extract_response.py b/dendrite/models/response/extract_response.py index 1d0035b..168e90a 100644 --- a/dendrite/models/response/extract_response.py +++ b/dendrite/models/response/extract_response.py @@ -6,11 +6,10 @@ T = TypeVar("T") + class ExtractResponse(BaseModel, Generic[T]): status: Status message: str return_data: Optional[T] = None used_cache: bool = False created_script: Optional[str] = None - - diff --git a/dendrite/models/response/get_element_response.py b/dendrite/models/response/get_element_response.py index 2c8e07a..3491818 100644 --- a/dendrite/models/response/get_element_response.py +++ b/dendrite/models/response/get_element_response.py @@ -11,5 +11,3 @@ class GetElementResponse(BaseModel): selectors: Optional[Union[List[str], Dict[str, List[str]]]] = None message: str = "" used_cache: bool = False - - diff --git a/dendrite/models/scripts.py b/dendrite/models/scripts.py index 3d8ee61..7507d9d 100644 --- a/dendrite/models/scripts.py +++ b/dendrite/models/scripts.py @@ -1,5 +1,6 @@ from pydantic import BaseModel + class Script(BaseModel): url: str domain: str diff --git a/dendrite/models/selector.py b/dendrite/models/selector.py index a84feb1..5e8e964 100644 --- a/dendrite/models/selector.py +++ b/dendrite/models/selector.py @@ -1,5 +1,6 @@ from pydantic import BaseModel + class Selector(BaseModel): selector: str prompt: str diff --git a/dendrite/models/status.py b/dendrite/models/status.py index 8d5beac..0068d7d 100644 --- a/dendrite/models/status.py +++ b/dendrite/models/status.py @@ -1,4 +1,4 @@ from typing import Literal -Status = Literal["success", "failed", "loading", "impossible"] \ No newline at end of file +Status = Literal["success", "failed", "loading", "impossible"] From 869605436f62ee99666714cd1bd12454796ff1f2 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Mon, 2 Dec 2024 11:14:59 +0100 Subject: [PATCH 06/18] update sync sdk generation --- dendrite/__init__.py | 11 + .../async_api/_core/dendrite_browser.py | 9 +- .../async_api/_core/dendrite_element.py | 4 +- .../browser/async_api/_core/dendrite_page.py | 6 +- .../async_api/_core/protocol/page_protocol.py | 4 +- dendrite/browser/sync_api/__init__.py | 7 + dendrite/browser/sync_api/_core/__init__.py | 0 .../browser/sync_api/_core/_impl_browser.py | 61 +++ .../browser/sync_api/_core/_impl_mapping.py | 29 ++ .../browser/sync_api/_core/_js/__init__.py | 11 + .../sync_api/_core/_js/eventListenerPatch.js | 90 ++++ .../sync_api/_core/_js/generateDendriteIDs.js | 88 ++++ .../_core/_js/generateDendriteIDsIframe.js | 93 +++++ .../sync_api/_core/_local_browser_impl.py | 27 ++ .../sync_api/_core/_managers/__init__.py | 0 .../_core/_managers/navigation_tracker.py | 67 +++ .../sync_api/_core/_managers/page_manager.py | 74 ++++ .../_core/_managers/screenshot_manager.py | 50 +++ dendrite/browser/sync_api/_core/_type_spec.py | 35 ++ dendrite/browser/sync_api/_core/_utils.py | 104 +++++ .../sync_api/_core/dendrite_browser.py | 393 ++++++++++++++++++ .../sync_api/_core/dendrite_element.py | 239 +++++++++++ .../browser/sync_api/_core/dendrite_page.py | 377 +++++++++++++++++ dendrite/browser/sync_api/_core/event_sync.py | 45 ++ .../browser/sync_api/_core/mixin/__init__.py | 21 + dendrite/browser/sync_api/_core/mixin/ask.py | 189 +++++++++ .../browser/sync_api/_core/mixin/click.py | 56 +++ .../browser/sync_api/_core/mixin/extract.py | 211 ++++++++++ .../sync_api/_core/mixin/fill_fields.py | 76 ++++ .../sync_api/_core/mixin/get_element.py | 286 +++++++++++++ .../browser/sync_api/_core/mixin/keyboard.py | 62 +++ .../browser/sync_api/_core/mixin/markdown.py | 23 + .../sync_api/_core/mixin/screenshot.py | 20 + .../browser/sync_api/_core/mixin/wait_for.py | 53 +++ .../browser/sync_api/_core/models/__init__.py | 0 .../sync_api/_core/models/authentication.py | 47 +++ .../_core/models/download_interface.py | 20 + .../_core/models/page_diff_information.py | 0 .../sync_api/_core/models/page_information.py | 15 + .../browser/sync_api/_core/models/response.py | 54 +++ .../sync_api/_core/protocol/page_protocol.py | 20 + dendrite/browser/sync_api/_dom/__init__.py | 0 .../browser/sync_api/_remote_impl/__init__.py | 3 + .../_remote_impl/browserbase/__init__.py | 3 + .../_remote_impl/browserbase/_client.py | 63 +++ .../_remote_impl/browserbase/_download.py | 53 +++ .../_remote_impl/browserbase/_impl.py | 64 +++ .../_remote_impl/browserless/__init__.py | 0 .../_remote_impl/browserless/_impl.py | 57 +++ dendrite/logic/factory.py | 2 +- dendrite/logic/interfaces/__init__.py | 8 + dendrite/logic/interfaces/async_api.py | 2 +- dendrite/logic/interfaces/sync_api.py | 2 +- dendrite/logic/llm/config.py | 12 +- poetry.lock | 23 +- pyproject.toml | 6 +- scripts/generate_sync.py | 5 +- 57 files changed, 3254 insertions(+), 26 deletions(-) create mode 100644 dendrite/browser/sync_api/__init__.py create mode 100644 dendrite/browser/sync_api/_core/__init__.py create mode 100644 dendrite/browser/sync_api/_core/_impl_browser.py create mode 100644 dendrite/browser/sync_api/_core/_impl_mapping.py create mode 100644 dendrite/browser/sync_api/_core/_js/__init__.py create mode 100644 dendrite/browser/sync_api/_core/_js/eventListenerPatch.js create mode 100644 dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js create mode 100644 dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js create mode 100644 dendrite/browser/sync_api/_core/_local_browser_impl.py create mode 100644 dendrite/browser/sync_api/_core/_managers/__init__.py create mode 100644 dendrite/browser/sync_api/_core/_managers/navigation_tracker.py create mode 100644 dendrite/browser/sync_api/_core/_managers/page_manager.py create mode 100644 dendrite/browser/sync_api/_core/_managers/screenshot_manager.py create mode 100644 dendrite/browser/sync_api/_core/_type_spec.py create mode 100644 dendrite/browser/sync_api/_core/_utils.py create mode 100644 dendrite/browser/sync_api/_core/dendrite_browser.py create mode 100644 dendrite/browser/sync_api/_core/dendrite_element.py create mode 100644 dendrite/browser/sync_api/_core/dendrite_page.py create mode 100644 dendrite/browser/sync_api/_core/event_sync.py create mode 100644 dendrite/browser/sync_api/_core/mixin/__init__.py create mode 100644 dendrite/browser/sync_api/_core/mixin/ask.py create mode 100644 dendrite/browser/sync_api/_core/mixin/click.py create mode 100644 dendrite/browser/sync_api/_core/mixin/extract.py create mode 100644 dendrite/browser/sync_api/_core/mixin/fill_fields.py create mode 100644 dendrite/browser/sync_api/_core/mixin/get_element.py create mode 100644 dendrite/browser/sync_api/_core/mixin/keyboard.py create mode 100644 dendrite/browser/sync_api/_core/mixin/markdown.py create mode 100644 dendrite/browser/sync_api/_core/mixin/screenshot.py create mode 100644 dendrite/browser/sync_api/_core/mixin/wait_for.py create mode 100644 dendrite/browser/sync_api/_core/models/__init__.py create mode 100644 dendrite/browser/sync_api/_core/models/authentication.py create mode 100644 dendrite/browser/sync_api/_core/models/download_interface.py create mode 100644 dendrite/browser/sync_api/_core/models/page_diff_information.py create mode 100644 dendrite/browser/sync_api/_core/models/page_information.py create mode 100644 dendrite/browser/sync_api/_core/models/response.py create mode 100644 dendrite/browser/sync_api/_core/protocol/page_protocol.py create mode 100644 dendrite/browser/sync_api/_dom/__init__.py create mode 100644 dendrite/browser/sync_api/_remote_impl/__init__.py create mode 100644 dendrite/browser/sync_api/_remote_impl/browserbase/__init__.py create mode 100644 dendrite/browser/sync_api/_remote_impl/browserbase/_client.py create mode 100644 dendrite/browser/sync_api/_remote_impl/browserbase/_download.py create mode 100644 dendrite/browser/sync_api/_remote_impl/browserbase/_impl.py create mode 100644 dendrite/browser/sync_api/_remote_impl/browserless/__init__.py create mode 100644 dendrite/browser/sync_api/_remote_impl/browserless/_impl.py create mode 100644 dendrite/logic/interfaces/__init__.py diff --git a/dendrite/__init__.py b/dendrite/__init__.py index 9d84deb..979e742 100644 --- a/dendrite/__init__.py +++ b/dendrite/__init__.py @@ -7,6 +7,13 @@ AsyncElementsResponse, ) +from dendrite.browser.sync_api import ( + Dendrite, + Element, + Page, + ElementsResponse, +) + # logger.remove() # fmt = "{time: HH:mm:ss.SSS} | {level: <8}- {message}" @@ -19,4 +26,8 @@ "AsyncElement", "AsyncPage", "AsyncElementsResponse", + "Dendrite", + "Element", + "Page", + "ElementsResponse", ] diff --git a/dendrite/browser/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/_core/dendrite_browser.py index 4bb6e40..6bd4b00 100644 --- a/dendrite/browser/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/_core/dendrite_browser.py @@ -41,7 +41,7 @@ WaitForMixin, ) from dendrite.browser.remote import Providers -from dendrite.logic.interfaces.async_api import LocalProtocol, LogicAPIProtocol +from dendrite.logic.interfaces import AsyncProtocol class AsyncDendrite( @@ -81,9 +81,6 @@ class AsyncDendrite( def __init__( self, - dendrite_api_key: Optional[str] = None, - openai_api_key: Optional[str] = None, - anthropic_api_key: Optional[str] = None, playwright_options: Any = { "headless": False, "args": STEALTH_ARGS, @@ -122,7 +119,7 @@ def __init__( self._upload_handler = EventSync(event_type=FileChooser) self._download_handler = EventSync(event_type=Download) self.closed = False - self._browser_api_client: LogicAPIProtocol = LocalProtocol() + self._browser_api_client: AsyncProtocol = AsyncProtocol() @property def pages(self) -> List[AsyncPage]: @@ -141,7 +138,7 @@ async def _get_page(self) -> AsyncPage: active_page = await self.get_active_page() return active_page - def _get_logic_api(self) -> "LogicAPIProtocol": + def _get_logic_api(self) -> AsyncProtocol: return self._browser_api_client def _get_dendrite_browser(self) -> "AsyncDendrite": diff --git a/dendrite/browser/async_api/_core/dendrite_element.py b/dendrite/browser/async_api/_core/dendrite_element.py index 5af8fdc..cabec7f 100644 --- a/dendrite/browser/async_api/_core/dendrite_element.py +++ b/dendrite/browser/async_api/_core/dendrite_element.py @@ -13,7 +13,7 @@ IncorrectOutcomeError, ) -from dendrite.logic.interfaces.async_api import LogicAPIProtocol +from dendrite.logic.interfaces import AsyncProtocol if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite @@ -109,7 +109,7 @@ def __init__( dendrite_id: str, locator: Locator, dendrite_browser: AsyncDendrite, - browser_api_client: LogicAPIProtocol, + browser_api_client: AsyncProtocol, ): """ Initialize a AsyncElement. diff --git a/dendrite/browser/async_api/_core/dendrite_page.py b/dendrite/browser/async_api/_core/dendrite_page.py index 70190c3..60bb570 100644 --- a/dendrite/browser/async_api/_core/dendrite_page.py +++ b/dendrite/browser/async_api/_core/dendrite_page.py @@ -20,7 +20,7 @@ from dendrite.browser.async_api._core.mixin.markdown import MarkdownMixin from dendrite.browser.async_api._core.mixin.wait_for import WaitForMixin -from dendrite.logic.interfaces.async_api import LogicAPIProtocol +from dendrite.logic.interfaces import AsyncProtocol from dendrite.models.page_information import PageInformation if TYPE_CHECKING: @@ -54,7 +54,7 @@ def __init__( self, page: PlaywrightPage, dendrite_browser: "AsyncDendrite", - browser_api_client: "LogicAPIProtocol", + browser_api_client: AsyncProtocol, ): self.playwright_page = page self.screenshot_manager = ScreenshotManager(page) @@ -96,7 +96,7 @@ async def _get_page(self) -> "AsyncPage": def _get_dendrite_browser(self) -> "AsyncDendrite": return self.dendrite_browser - def _get_logic_api(self) -> LogicAPIProtocol: + def _get_logic_api(self) -> AsyncProtocol: return self._browser_api_client async def goto( diff --git a/dendrite/browser/async_api/_core/protocol/page_protocol.py b/dendrite/browser/async_api/_core/protocol/page_protocol.py index 915f0fc..afe51cd 100644 --- a/dendrite/browser/async_api/_core/protocol/page_protocol.py +++ b/dendrite/browser/async_api/_core/protocol/page_protocol.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Protocol from dendrite.logic.hosted._api.browser_api_client import BrowserAPIClient -from dendrite.logic.interfaces.async_api import LocalProtocol +from dendrite.logic.interfaces import AsyncProtocol if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite @@ -16,6 +16,6 @@ class DendritePageProtocol(Protocol): def _get_dendrite_browser(self) -> "AsyncDendrite": ... - def _get_logic_api(self) -> LocalProtocol: ... + def _get_logic_api(self) -> AsyncProtocol: ... async def _get_page(self) -> "AsyncPage": ... diff --git a/dendrite/browser/sync_api/__init__.py b/dendrite/browser/sync_api/__init__.py new file mode 100644 index 0000000..3085e23 --- /dev/null +++ b/dendrite/browser/sync_api/__init__.py @@ -0,0 +1,7 @@ +from loguru import logger +from ._core.dendrite_browser import Dendrite +from ._core.dendrite_element import Element +from ._core.dendrite_page import Page +from ._core.models.response import ElementsResponse + +__all__ = ["Dendrite", "Element", "Page", "ElementsResponse"] diff --git a/dendrite/browser/sync_api/_core/__init__.py b/dendrite/browser/sync_api/_core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/_core/_impl_browser.py b/dendrite/browser/sync_api/_core/_impl_browser.py new file mode 100644 index 0000000..b1d39a0 --- /dev/null +++ b/dendrite/browser/sync_api/_core/_impl_browser.py @@ -0,0 +1,61 @@ +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from playwright.sync_api import Browser, Download, Playwright +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage + + +class ImplBrowser(ABC): + + @abstractmethod + def __init__(self, settings): + pass + + @abstractmethod + def get_download( + self, dendrite_browser: "Dendrite", pw_page: PlaywrightPage, timeout: float + ) -> Download: + """ + Retrieves the download event from the browser. + + Returns: + Download: The download event. + + Raises: + Exception: If there is an issue retrieving the download event. + """ + + @abstractmethod + def start_browser(self, playwright: Playwright, pw_options: dict) -> Browser: + """ + Starts the browser session. + + Returns: + Browser: The browser session. + + Raises: + Exception: If there is an issue starting the browser session. + """ + + @abstractmethod + def configure_context(self, browser: "Dendrite") -> None: + """ + Configures the browser context. + + Args: + browser (Dendrite): The browser to configure. + + Raises: + Exception: If there is an issue configuring the browser context. + """ + + @abstractmethod + def stop_session(self) -> None: + """ + Stops the browser session. + + Raises: + Exception: If there is an issue stopping the browser session. + """ diff --git a/dendrite/browser/sync_api/_core/_impl_mapping.py b/dendrite/browser/sync_api/_core/_impl_mapping.py new file mode 100644 index 0000000..227b13e --- /dev/null +++ b/dendrite/browser/sync_api/_core/_impl_mapping.py @@ -0,0 +1,29 @@ +from typing import Dict, Optional, Type +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser +from dendrite.browser.sync_api._core._local_browser_impl import LocalImpl +from dendrite.browser.sync_api._remote_impl.browserbase._impl import BrowserBaseImpl +from dendrite.browser.sync_api._remote_impl.browserless._impl import BrowserlessImpl +from dendrite.browser.remote import Providers +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser.remote.browserless_config import BrowserlessConfig + +IMPL_MAPPING: Dict[Type[Providers], Type[ImplBrowser]] = { + BrowserbaseConfig: BrowserBaseImpl, + BrowserlessConfig: BrowserlessImpl, +} +SETTINGS_CLASSES: Dict[str, Type[Providers]] = { + "browserbase": BrowserbaseConfig, + "browserless": BrowserlessConfig, +} + + +def get_impl(remote_provider: Optional[Providers]) -> ImplBrowser: + if remote_provider is None: + return LocalImpl() + try: + provider_class = IMPL_MAPPING[type(remote_provider)] + except KeyError: + raise ValueError( + f"No implementation for {type(remote_provider)}. Available providers: {', '.join(map(lambda x: x.__name__, IMPL_MAPPING.keys()))}" + ) + return provider_class(remote_provider) diff --git a/dendrite/browser/sync_api/_core/_js/__init__.py b/dendrite/browser/sync_api/_core/_js/__init__.py new file mode 100644 index 0000000..ccaf080 --- /dev/null +++ b/dendrite/browser/sync_api/_core/_js/__init__.py @@ -0,0 +1,11 @@ +from pathlib import Path + + +def load_script(filename: str) -> str: + current_dir = Path(__file__).parent + file_path = current_dir / filename + return file_path.read_text(encoding="utf-8") + + +GENERATE_DENDRITE_IDS_SCRIPT = load_script("generateDendriteIDs.js") +GENERATE_DENDRITE_IDS_IFRAME_SCRIPT = load_script("generateDendriteIDsIframe.js") diff --git a/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js b/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js new file mode 100644 index 0000000..7f03d55 --- /dev/null +++ b/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js @@ -0,0 +1,90 @@ +// Save the original methods before redefining them +EventTarget.prototype._originalAddEventListener = EventTarget.prototype.addEventListener; +EventTarget.prototype._originalRemoveEventListener = EventTarget.prototype.removeEventListener; + +// Redefine the addEventListener method +EventTarget.prototype.addEventListener = function(event, listener, options = false) { + // Initialize the eventListenerList if it doesn't exist + if (!this.eventListenerList) { + this.eventListenerList = {}; + } + // Initialize the event list for the specific event if it doesn't exist + if (!this.eventListenerList[event]) { + this.eventListenerList[event] = []; + } + // Add the event listener details to the event list + this.eventListenerList[event].push({ listener, options, outerHTML: this.outerHTML }); + + // Call the original addEventListener method + this._originalAddEventListener(event, listener, options); +}; + +// Redefine the removeEventListener method +EventTarget.prototype.removeEventListener = function(event, listener, options = false) { + // Remove the event listener details from the event list + if (this.eventListenerList && this.eventListenerList[event]) { + this.eventListenerList[event] = this.eventListenerList[event].filter( + item => item.listener !== listener + ); + } + + // Call the original removeEventListener method + this._originalRemoveEventListener( event, listener, options); +}; + +// Get event listeners for a specific event type or all events if not specified +EventTarget.prototype._getEventListeners = function(eventType) { + if (!this.eventListenerList) { + this.eventListenerList = {}; + } + + const eventsToCheck = ['click', 'dblclick', 'mousedown', 'mouseup', 'mouseover', 'mouseout', 'mousemove', 'keydown', 'keyup', 'keypress']; + + eventsToCheck.forEach(type => { + if (!eventType || eventType === type) { + if (this[`on${type}`]) { + if (!this.eventListenerList[type]) { + this.eventListenerList[type] = []; + } + this.eventListenerList[type].push({ listener: this[`on${type}`], inline: true }); + } + } + }); + + return eventType === undefined ? this.eventListenerList : this.eventListenerList[eventType]; +}; + +// Utility to show events +function _showEvents(events) { + let result = ''; + for (let event in events) { + result += `${event} ----------------> ${events[event].length}\n`; + for (let listenerObj of events[event]) { + result += `${listenerObj.listener.toString()}\n`; + } + } + return result; +} + +// Extend EventTarget prototype with utility methods +EventTarget.prototype.on = function(event, callback, options) { + this.addEventListener(event, callback, options); + return this; +}; + +EventTarget.prototype.off = function(event, callback, options) { + this.removeEventListener(event, callback, options); + return this; +}; + +EventTarget.prototype.emit = function(event, args = null) { + this.dispatchEvent(new CustomEvent(event, { detail: args })); + return this; +}; + +// Make these methods non-enumerable +Object.defineProperties(EventTarget.prototype, { + on: { enumerable: false }, + off: { enumerable: false }, + emit: { enumerable: false } +}); diff --git a/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js b/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js new file mode 100644 index 0000000..0ae5f61 --- /dev/null +++ b/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js @@ -0,0 +1,88 @@ +var hashCode = (str) => { + var hash = 0, i, chr; + if (str.length === 0) return hash; + for (i = 0; i < str.length; i++) { + chr = str.charCodeAt(i); + hash = ((hash << 5) - hash) + chr; + hash |= 0; // Convert to 32bit integer + } + return hash; +} + + +const getElementIndex = (element) => { + let index = 1; + let sibling = element.previousElementSibling; + + while (sibling) { + if (sibling.localName === element.localName) { + index++; + } + sibling = sibling.previousElementSibling; + } + + return index; +}; + + +const segs = function elmSegs(elm) { + if (!elm || elm.nodeType !== 1) return ['']; + if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; + const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; + let index = getElementIndex(elm); + + return [...elmSegs(elm.parentNode), `${localName}[${index}]`]; +}; + +var getXPathForElement = (element) => { + return segs(element).join('/'); +} + +// Create a Map to store used hashes and their counters +const usedHashes = new Map(); + +var markHidden = (hidden_element) => { + // Mark the hidden element itself + hidden_element.setAttribute('data-hidden', 'true'); + +} + +document.querySelectorAll('*').forEach((element, index) => { + try { + + const xpath = getXPathForElement(element); + const hash = hashCode(xpath); + const baseId = hash.toString(36); + + // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; + const isHidden = !element.checkVisibility(); + // computedStyle.width === '0px' || + // computedStyle.height === '0px'; + + if (isHidden) { + markHidden(element); + }else{ + element.removeAttribute("data-hidden") // in case we hid it in a previous call + } + + let uniqueId = baseId; + let counter = 0; + + // Check if this hash has been used before + while (usedHashes.has(uniqueId)) { + // If it has, increment the counter and create a new uniqueId + counter++; + uniqueId = `${baseId}_${counter}`; + } + + // Add the uniqueId to the usedHashes Map + usedHashes.set(uniqueId, true); + element.setAttribute('d-id', uniqueId); + } catch (error) { + // Fallback: use a hash of the tag name and index + const fallbackId = hashCode(`${element.tagName}_${index}`).toString(36); + console.error('Error processing element, using fallback:',fallbackId, element, error); + + element.setAttribute('d-id', `fallback_${fallbackId}`); + } +}); \ No newline at end of file diff --git a/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js b/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js new file mode 100644 index 0000000..4f59cef --- /dev/null +++ b/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js @@ -0,0 +1,93 @@ +({frame_path}) => { + var hashCode = (str) => { + var hash = 0, i, chr; + if (str.length === 0) return hash; + for (i = 0; i < str.length; i++) { + chr = str.charCodeAt(i); + hash = ((hash << 5) - hash) + chr; + hash |= 0; // Convert to 32bit integer + } + return hash; + } + + const getElementIndex = (element) => { + let index = 1; + let sibling = element.previousElementSibling; + + while (sibling) { + if (sibling.localName === element.localName) { + index++; + } + sibling = sibling.previousElementSibling; + } + + return index; + }; + + + const segs = function elmSegs(elm) { + if (!elm || elm.nodeType !== 1) return ['']; + if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; + const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; + let index = getElementIndex(elm); + + return [...elmSegs(elm.parentNode), `${localName}[${index}]`]; + }; + + var getXPathForElement = (element) => { + return segs(element).join('/'); + } + + // Create a Map to store used hashes and their counters + const usedHashes = new Map(); + + var markHidden = (hidden_element) => { + // Mark the hidden element itself + hidden_element.setAttribute('data-hidden', 'true'); + } + + document.querySelectorAll('*').forEach((element, index) => { + try { + + + // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; + const isHidden = !element.checkVisibility(); + // computedStyle.width === '0px' || + // computedStyle.height === '0px'; + + if (isHidden) { + markHidden(element); + }else{ + element.removeAttribute("data-hidden") // in case we hid it in a previous call + } + let xpath = getXPathForElement(element); + if(frame_path){ + element.setAttribute("iframe-path",frame_path) + xpath = frame_path + xpath; + } + const hash = hashCode(xpath); + const baseId = hash.toString(36); + + let uniqueId = baseId; + let counter = 0; + + // Check if this hash has been used before + while (usedHashes.has(uniqueId)) { + // If it has, increment the counter and create a new uniqueId + counter++; + uniqueId = `${baseId}_${counter}`; + } + + // Add the uniqueId to the usedHashes Map + usedHashes.set(uniqueId, true); + element.setAttribute('d-id', uniqueId); + } catch (error) { + // Fallback: use a hash of the tag name and index + const fallbackId = hashCode(`${element.tagName}_${index}`).toString(36); + console.error('Error processing element, using fallback:',fallbackId, element, error); + + element.setAttribute('d-id', `fallback_${fallbackId}`); + } + }); +} + diff --git a/dendrite/browser/sync_api/_core/_local_browser_impl.py b/dendrite/browser/sync_api/_core/_local_browser_impl.py new file mode 100644 index 0000000..464a90e --- /dev/null +++ b/dendrite/browser/sync_api/_core/_local_browser_impl.py @@ -0,0 +1,27 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from playwright.sync_api import Browser, Download, Playwright +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage + + +class LocalImpl(ImplBrowser): + + def __init__(self) -> None: + pass + + def start_browser(self, playwright: Playwright, pw_options) -> Browser: + return playwright.chromium.launch(**pw_options) + + def get_download( + self, dendrite_browser: "Dendrite", pw_page: PlaywrightPage, timeout: float + ) -> Download: + return dendrite_browser._download_handler.get_data(pw_page, timeout) + + def configure_context(self, browser: "Dendrite"): + pass + + def stop_session(self): + pass diff --git a/dendrite/browser/sync_api/_core/_managers/__init__.py b/dendrite/browser/sync_api/_core/_managers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py b/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py new file mode 100644 index 0000000..8dc632b --- /dev/null +++ b/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py @@ -0,0 +1,67 @@ +import time +import time +from typing import TYPE_CHECKING, Dict, Optional + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_page import Page + + +class NavigationTracker: + + def __init__(self, page: "Page"): + self.playwright_page = page.playwright_page + self._nav_start_timestamp: Optional[float] = None + self.playwright_page.on("framenavigated", self._on_frame_navigated) + self.playwright_page.on("popup", self._on_popup) + self._last_events: Dict[str, Optional[float]] = { + "framenavigated": None, + "popup": None, + } + + def _on_frame_navigated(self, frame): + self._last_events["framenavigated"] = time.time() + if frame is self.playwright_page.main_frame: + self._last_main_frame_url = frame.url + self._last_frame_navigated_timestamp = time.time() + + def _on_popup(self, page): + self._last_events["popup"] = time.time() + + def start_nav_tracking(self): + """Call this just before performing an action that might trigger navigation""" + self._nav_start_timestamp = time.time() + for event in self._last_events: + self._last_events[event] = None + + def get_nav_events_since_start(self): + """ + Returns which events have fired since start_nav_tracking() was called + and how long after the start they occurred + """ + if self._nav_start_timestamp is None: + return "Navigation tracking not started. Call start_nav_tracking() first." + results = {} + for event, timestamp in self._last_events.items(): + if timestamp is not None: + delay = timestamp - self._nav_start_timestamp + results[event] = f"{delay:.3f}s" + else: + results[event] = "not fired" + return results + + def has_navigated_since_start(self): + """Returns True if any navigation event has occurred since start_nav_tracking()""" + if self._nav_start_timestamp is None: + return False + start_time = time.time() + max_wait = 1.0 + while time.time() - start_time < max_wait: + if any( + ( + timestamp is not None and timestamp > self._nav_start_timestamp + for timestamp in self._last_events.values() + ) + ): + return True + time.sleep(0.1) + return False diff --git a/dendrite/browser/sync_api/_core/_managers/page_manager.py b/dendrite/browser/sync_api/_core/_managers/page_manager.py new file mode 100644 index 0000000..ad25fa6 --- /dev/null +++ b/dendrite/browser/sync_api/_core/_managers/page_manager.py @@ -0,0 +1,74 @@ +from typing import TYPE_CHECKING, Optional +from loguru import logger +from playwright.sync_api import BrowserContext, Download, FileChooser + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core.dendrite_page import Page + + +class PageManager: + + def __init__(self, dendrite_browser, browser_context: BrowserContext): + self.pages: list[Page] = [] + self.active_page: Optional[Page] = None + self.browser_context = browser_context + self.dendrite_browser: Dendrite = dendrite_browser + browser_context.on("page", self._page_on_open_handler) + + def new_page(self) -> Page: + new_page = self.browser_context.new_page() + if self.active_page and new_page == self.active_page.playwright_page: + return self.active_page + client = self.dendrite_browser._get_logic_api() + dendrite_page = Page(new_page, self.dendrite_browser, client) + self.pages.append(dendrite_page) + self.active_page = dendrite_page + return dendrite_page + + def get_active_page(self) -> Page: + if self.active_page is None: + return self.new_page() + return self.active_page + + def _page_on_close_handler(self, page: PlaywrightPage): + if self.browser_context and (not self.dendrite_browser.closed): + copy_pages = self.pages.copy() + is_active_page = False + for dendrite_page in copy_pages: + if dendrite_page.playwright_page == page: + self.pages.remove(dendrite_page) + if dendrite_page == self.active_page: + is_active_page = True + break + for i in reversed(range(len(self.pages))): + try: + self.active_page = self.pages[i] + self.pages[i].playwright_page.bring_to_front() + break + except Exception as e: + logger.warning(f"Error switching to the next page: {e}") + continue + + def _page_on_crash_handler(self, page: PlaywrightPage): + logger.error(f"Page crashed: {page.url}") + page.reload() + + def _page_on_download_handler(self, download: Download): + logger.debug(f"Download started: {download.url}") + self.dendrite_browser._download_handler.set_event(download) + + def _page_on_filechooser_handler(self, file_chooser: FileChooser): + logger.debug("File chooser opened") + self.dendrite_browser._upload_handler.set_event(file_chooser) + + def _page_on_open_handler(self, page: PlaywrightPage): + page.on("close", self._page_on_close_handler) + page.on("crash", self._page_on_crash_handler) + page.on("download", self._page_on_download_handler) + page.on("filechooser", self._page_on_filechooser_handler) + client = self.dendrite_browser._get_logic_api() + dendrite_page = Page(page, self.dendrite_browser, client) + self.pages.append(dendrite_page) + self.active_page = dendrite_page diff --git a/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py b/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py new file mode 100644 index 0000000..9a01f7c --- /dev/null +++ b/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py @@ -0,0 +1,50 @@ +import base64 +import os +from uuid import uuid4 +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage + + +class ScreenshotManager: + + def __init__(self, page: PlaywrightPage) -> None: + self.screenshot_before: str = "" + self.screenshot_after: str = "" + self.page = page + + def take_full_page_screenshot(self) -> str: + try: + scroll_height = self.page.evaluate( + "\n () => {\n const body = document.body;\n if (!body) {\n return 0; // Return 0 if body is null\n }\n return body.scrollHeight || 0;\n }\n " + ) + if scroll_height > 30000: + print( + f"Page height ({scroll_height}px) exceeds 30000px. Taking viewport screenshot instead." + ) + return self.take_viewport_screenshot() + image_data = self.page.screenshot( + type="jpeg", full_page=True, timeout=10000 + ) + except Exception as e: + print( + f"Full-page screenshot failed: {e}. Falling back to viewport screenshot." + ) + return self.take_viewport_screenshot() + if image_data is None: + return "" + return base64.b64encode(image_data).decode("utf-8") + + def take_viewport_screenshot(self) -> str: + image_data = self.page.screenshot(type="jpeg", timeout=10000) + if image_data is None: + return "" + reduced_base64 = base64.b64encode(image_data).decode("utf-8") + return reduced_base64 + + def store_screenshot(self, name, image_data): + if not name: + name = str(uuid4()) + filepath = os.path.join("test", f"{name}.jpeg") + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "wb") as file: + file.write(image_data) + return filepath diff --git a/dendrite/browser/sync_api/_core/_type_spec.py b/dendrite/browser/sync_api/_core/_type_spec.py new file mode 100644 index 0000000..7584120 --- /dev/null +++ b/dendrite/browser/sync_api/_core/_type_spec.py @@ -0,0 +1,35 @@ +import inspect +from typing import Any, Dict, Literal, Type, TypeVar, Union +from playwright.sync_api import Page +from pydantic import BaseModel + +Interaction = Literal["click", "fill", "hover"] +T = TypeVar("T") +PydanticModel = TypeVar("PydanticModel", bound=BaseModel) +PrimitiveTypes = PrimitiveTypes = Union[Type[bool], Type[int], Type[float], Type[str]] +JsonSchema = Dict[str, Any] +TypeSpec = Union[PrimitiveTypes, PydanticModel, JsonSchema] +PlaywrightPage = Page + + +def to_json_schema(type_spec: TypeSpec) -> Dict[str, Any]: + if isinstance(type_spec, dict): + return type_spec + if inspect.isclass(type_spec) and issubclass(type_spec, BaseModel): + return type_spec.model_json_schema() + if type_spec in (bool, int, float, str): + type_map = {bool: "boolean", int: "integer", float: "number", str: "string"} + return {"type": type_map[type_spec]} + raise ValueError(f"Unsupported type specification: {type_spec}") + + +def convert_to_type_spec(type_spec: TypeSpec, return_data: Any) -> TypeSpec: + if isinstance(type_spec, type): + if issubclass(type_spec, BaseModel): + return type_spec.model_validate(return_data) + if type_spec in (str, float, bool, int): + return type_spec(return_data) + raise ValueError(f"Unsupported type: {type_spec}") + if isinstance(type_spec, dict): + return return_data + raise ValueError(f"Unsupported type specification: {type_spec}") diff --git a/dendrite/browser/sync_api/_core/_utils.py b/dendrite/browser/sync_api/_core/_utils.py new file mode 100644 index 0000000..37f328e --- /dev/null +++ b/dendrite/browser/sync_api/_core/_utils.py @@ -0,0 +1,104 @@ +from typing import TYPE_CHECKING, List, Optional, Union +from bs4 import BeautifulSoup +from loguru import logger +from playwright.sync_api import ElementHandle, Error, Frame, FrameLocator +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core.dendrite_element import Element +from dendrite.browser.sync_api._core.models.response import ElementsResponse +from dendrite.models.response.get_element_response import GetElementResponse + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_page import Page +from dendrite.browser.sync_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT +from dendrite.logic.dom.strip import mild_strip_in_place + + +def expand_iframes(page: PlaywrightPage, page_soup: BeautifulSoup): + + def get_iframe_path(frame: Frame): + path_parts = [] + current_frame = frame + while current_frame.parent_frame is not None: + iframe_element = current_frame.frame_element() + iframe_id = iframe_element.get_attribute("d-id") + if iframe_id is None: + return None + path_parts.insert(0, iframe_id) + current_frame = current_frame.parent_frame + return "|".join(path_parts) + + for frame in page.frames: + if frame.parent_frame is None: + continue + try: + iframe_element = frame.frame_element() + iframe_id = iframe_element.get_attribute("d-id") + if iframe_id is None: + continue + iframe_path = get_iframe_path(frame) + except Error as e: + continue + if iframe_path is None: + continue + try: + frame.evaluate( + GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, {"frame_path": iframe_path} + ) + frame_content = frame.content() + frame_tree = BeautifulSoup(frame_content, "lxml") + mild_strip_in_place(frame_tree) + merge_iframe_to_page(iframe_id, page_soup, frame_tree) + except Error as e: + logger.debug(f"Error processing frame {iframe_id}: {e}") + continue + + +def merge_iframe_to_page(iframe_id: str, page: BeautifulSoup, iframe: BeautifulSoup): + iframe_element = page.find("iframe", {"d-id": iframe_id}) + if iframe_element is None: + logger.debug(f"Could not find iframe with ID {iframe_id} in page soup") + return + iframe_element.replace_with(iframe) + + +def _get_all_elements_from_selector_soup( + selector: str, soup: BeautifulSoup, page: "Page" +) -> List[Element]: + dendrite_elements: List[Element] = [] + elements = soup.select(selector) + for element in elements: + frame = page._get_context(element) + d_id = element.get("d-id", "") + locator = frame.locator(f"xpath=//*[@d-id='{d_id}']") + if not d_id: + continue + if isinstance(d_id, list): + d_id = d_id[0] + dendrite_elements.append( + Element(d_id, locator, page.dendrite_browser, page._browser_api_client) + ) + return dendrite_elements + + +def get_elements_from_selectors_soup( + page: "Page", soup: BeautifulSoup, res: GetElementResponse, only_one: bool +) -> Union[Optional[Element], List[Element], ElementsResponse]: + if isinstance(res.selectors, dict): + result = {} + for key, selectors in res.selectors.items(): + for selector in selectors: + dendrite_elements = _get_all_elements_from_selector_soup( + selector, soup, page + ) + if len(dendrite_elements) > 0: + result[key] = dendrite_elements[0] + break + return ElementsResponse(result) + elif isinstance(res.selectors, list): + for selector in reversed(res.selectors): + dendrite_elements = _get_all_elements_from_selector_soup( + selector, soup, page + ) + if len(dendrite_elements) > 0: + return dendrite_elements[0] if only_one else dendrite_elements + return None diff --git a/dendrite/browser/sync_api/_core/dendrite_browser.py b/dendrite/browser/sync_api/_core/dendrite_browser.py new file mode 100644 index 0000000..4bf8316 --- /dev/null +++ b/dendrite/browser/sync_api/_core/dendrite_browser.py @@ -0,0 +1,393 @@ +import os +import pathlib +import re +from abc import ABC +from typing import Any, List, Optional, Sequence, Union +from uuid import uuid4 +from loguru import logger +from playwright.sync_api import ( + BrowserContext, + Download, + Error, + FileChooser, + FilePayload, + Playwright, + sync_playwright, +) +from dendrite.browser._common._exceptions.dendrite_exception import ( + BrowserNotLaunchedError, + DendriteException, + IncorrectOutcomeError, +) +from dendrite.browser._common.constants import STEALTH_ARGS +from dendrite.models.api_config import APIConfig +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser +from dendrite.browser.sync_api._core._impl_mapping import get_impl +from dendrite.browser.sync_api._core._managers.page_manager import PageManager +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core.dendrite_page import Page +from dendrite.browser.sync_api._core.event_sync import EventSync +from dendrite.browser.sync_api._core.mixin import ( + AskMixin, + ClickMixin, + ExtractionMixin, + FillFieldsMixin, + GetElementMixin, + KeyboardMixin, + MarkdownMixin, + ScreenshotMixin, + WaitForMixin, +) +from dendrite.browser.remote import Providers +from dendrite.logic.interfaces import SyncProtocol + + +class Dendrite( + ScreenshotMixin, + WaitForMixin, + MarkdownMixin, + ExtractionMixin, + AskMixin, + FillFieldsMixin, + ClickMixin, + KeyboardMixin, + GetElementMixin, + ABC, +): + """ + Dendrite is a class that manages a browser instance using Playwright, allowing + interactions with web pages using natural language. + + This class handles initialization with API keys for Dendrite, OpenAI, and Anthropic, manages browser + contexts, and provides methods for navigation, authentication, and other browser-related tasks. + + Attributes: + id (UUID): The unique identifier for the Dendrite instance. + auth_data (Optional[AuthSession]): The authentication session data for the browser. + dendrite_api_key (str): The API key for Dendrite, used for interactions with the Dendrite API. + playwright_options (dict): Options for configuring the Playwright browser instance. + playwright (Optional[Playwright]): The Playwright instance managing the browser. + browser_context (Optional[BrowserContext]): The current browser context, which may include cookies and other session data. + active_page_manager (Optional[PageManager]): The manager responsible for handling active pages within the browser context. + user_id (Optional[str]): The user ID associated with the browser session. + browser_api_client (BrowserAPIClient): The API client used for communicating with the Dendrite API. + api_config (APIConfig): The configuration for the language models, including API keys for OpenAI and Anthropic. + + Raises: + Exception: If any of the required API keys (Dendrite, OpenAI, Anthropic) are not provided or found in the environment variables. + """ + + def __init__( + self, + playwright_options: Any = {"headless": False, "args": STEALTH_ARGS}, + remote_config: Optional[Providers] = None, + ): + """ + Initializes Dendrite with API keys and Playwright options. + + Args: + dendrite_api_key (Optional[str]): The Dendrite API key. If not provided, it's fetched from the environment variables. + openai_api_key (Optional[str]): Your own OpenAI API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. + anthropic_api_key (Optional[str]): The own Anthropic API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. + playwright_options (Any): Options for configuring Playwright. Defaults to running in non-headless mode with stealth arguments. + + Raises: + MissingApiKeyError: If the Dendrite API key is not provided or found in the environment variables. + """ + self._impl = self._get_impl(remote_config) + self.playwright: Optional[Playwright] = None + self.browser_context: Optional[BrowserContext] = None + self._id = uuid4().hex + self._playwright_options = playwright_options + self._active_page_manager: Optional[PageManager] = None + self._user_id: Optional[str] = None + self._upload_handler = EventSync(event_type=FileChooser) + self._download_handler = EventSync(event_type=Download) + self.closed = False + self._browser_api_client: SyncProtocol = SyncProtocol() + + @property + def pages(self) -> List[Page]: + """ + Retrieves the list of active pages managed by the PageManager. + + Returns: + List[Page]: The list of active pages. + """ + if self._active_page_manager: + return self._active_page_manager.pages + else: + raise BrowserNotLaunchedError() + + def _get_page(self) -> Page: + active_page = self.get_active_page() + return active_page + + def _get_logic_api(self) -> SyncProtocol: + return self._browser_api_client + + def _get_dendrite_browser(self) -> "Dendrite": + return self + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def _get_impl(self, remote_provider: Optional[Providers]) -> ImplBrowser: + return get_impl(remote_provider) + + def get_active_page(self) -> Page: + """ + Retrieves the currently active page managed by the PageManager. + + Returns: + Page: The active page object. + + Raises: + Exception: If there is an issue retrieving the active page. + """ + active_page_manager = self._get_active_page_manager() + return active_page_manager.get_active_page() + + def new_tab( + self, url: str, timeout: Optional[float] = 15000, expected_page: str = "" + ) -> Page: + """ + Opens a new tab and navigates to the specified URL. + + Args: + url (str): The URL to navigate to. + timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. + expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. + + Returns: + Page: The page object after navigation. + + Raises: + Exception: If there is an error during navigation or if the expected page type is not found. + """ + return self.goto( + url, new_tab=True, timeout=timeout, expected_page=expected_page + ) + + def goto( + self, + url: str, + new_tab: bool = False, + timeout: Optional[float] = 15000, + expected_page: str = "", + ) -> Page: + """ + Navigates to the specified URL, optionally in a new tab + + Args: + url (str): The URL to navigate to. + new_tab (bool, optional): Whether to open the URL in a new tab. Defaults to False. + timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. + expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. + + Returns: + Page: The page object after navigation. + + Raises: + Exception: If there is an error during navigation or if the expected page type is not found. + """ + if not re.match("^\\w+://", url): + url = f"https://{url}" + active_page_manager = self._get_active_page_manager() + if new_tab: + active_page = active_page_manager.new_page() + else: + active_page = active_page_manager.get_active_page() + try: + logger.info(f"Going to {url}") + active_page.playwright_page.goto(url, timeout=timeout) + except TimeoutError: + logger.debug("Timeout when loading page but continuing anyways.") + except Exception as e: + logger.debug(f"Exception when loading page but continuing anyways. {e}") + if expected_page != "": + try: + prompt = f"We are checking if we have arrived on the expected type of page. If it is apparent that we have arrived on the wrong page, output an error. Here is the description: '{expected_page}'" + active_page.ask(prompt, bool) + except DendriteException as e: + raise IncorrectOutcomeError(f"Incorrect navigation, reason: {e}") + return active_page + + def scroll_to_bottom( + self, + timeout: float = 30000, + scroll_increment: int = 1000, + no_progress_limit: int = 3, + ): + """ + Scrolls to the bottom of the current page. + + Returns: + None + """ + active_page = self.get_active_page() + active_page.scroll_to_bottom( + timeout=timeout, + scroll_increment=scroll_increment, + no_progress_limit=no_progress_limit, + ) + + def _launch(self): + """ + Launches the Playwright instance and sets up the browser context and page manager. + + This method initializes the Playwright instance, creates a browser context, and sets up the PageManager. + It also applies any authentication data if available. + + Returns: + Tuple[Browser, BrowserContext, PageManager]: The launched browser, context, and page manager. + + Raises: + Exception: If there is an issue launching the browser or setting up the context. + """ + os.environ["PW_TEST_SCREENSHOT_NO_FONTS_READY"] = "1" + self._playwright = sync_playwright().start() + browser = self._impl.start_browser(self._playwright, self._playwright_options) + self.browser_context = ( + browser.contexts[0] if len(browser.contexts) > 0 else browser.new_context() + ) + self._active_page_manager = PageManager(self, self.browser_context) + self._impl.configure_context(self) + return (browser, self.browser_context, self._active_page_manager) + + def add_cookies(self, cookies): + """ + Adds cookies to the current browser context. + + Args: + cookies (List[Dict[str, Any]]): A list of cookies to be added to the browser context. + + Raises: + Exception: If the browser context is not initialized. + """ + if not self.browser_context: + raise DendriteException("Browser context not initialized") + self.browser_context.add_cookies(cookies) + + def close(self): + """ + Closes the browser and uploads authentication session data if available. + + This method stops the Playwright instance, closes the browser context + + Returns: + None + + Raises: + Exception: If there is an issue closing the browser or uploading session data. + """ + self.closed = True + try: + if self.browser_context: + self._impl.stop_session() + self.browser_context.close() + except Error: + pass + try: + if self._playwright: + self._playwright.stop() + except AttributeError: + pass + except Exception: + pass + + def _is_launched(self): + """ + Checks whether the browser context has been launched. + + Returns: + bool: True if the browser context is launched, False otherwise. + """ + return self.browser_context is not None + + def _get_active_page_manager(self) -> PageManager: + """ + Retrieves the active PageManager instance, launching the browser if necessary. + + Returns: + PageManager: The active PageManager instance. + + Raises: + Exception: If there is an issue launching the browser or retrieving the PageManager. + """ + if not self._active_page_manager: + (_, _, active_page_manager) = self._launch() + return active_page_manager + return self._active_page_manager + + def get_download(self, timeout: float) -> Download: + """ + Retrieves the download event from the browser. + + Returns: + Download: The download event. + + Raises: + Exception: If there is an issue retrieving the download event. + """ + active_page = self.get_active_page() + pw_page = active_page.playwright_page + return self._get_download(pw_page, timeout) + + def _get_download(self, pw_page: PlaywrightPage, timeout: float) -> Download: + """ + Retrieves the download event from the browser. + + Returns: + Download: The download event. + + Raises: + Exception: If there is an issue retrieving the download event. + """ + return self._download_handler.get_data(pw_page, timeout=timeout) + + def upload_files( + self, + files: Union[ + str, + pathlib.Path, + FilePayload, + Sequence[Union[str, pathlib.Path]], + Sequence[FilePayload], + ], + timeout: float = 30000, + ) -> None: + """ + Uploads files to the active page using a file chooser. + + Args: + files (Union[str, pathlib.Path, FilePayload, Sequence[Union[str, pathlib.Path]], Sequence[FilePayload]]): The file(s) to be uploaded. + This can be a file path, a `FilePayload` object, or a sequence of file paths or `FilePayload` objects. + timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the file chooser to be ready. Defaults to 30. + + Returns: + None + """ + page = self.get_active_page() + file_chooser = self._get_filechooser(page.playwright_page, timeout) + file_chooser.set_files(files) + + def _get_filechooser( + self, pw_page: PlaywrightPage, timeout: float = 30000 + ) -> FileChooser: + """ + Uploads files to the browser. + + Args: + timeout (float): The maximum time to wait for the file chooser dialog. Defaults to 30000 milliseconds. + + Returns: + FileChooser: The file chooser dialog. + + Raises: + Exception: If there is an issue uploading files. + """ + return self._upload_handler.get_data(pw_page, timeout=timeout) diff --git a/dendrite/browser/sync_api/_core/dendrite_element.py b/dendrite/browser/sync_api/_core/dendrite_element.py new file mode 100644 index 0000000..cf9cfe5 --- /dev/null +++ b/dendrite/browser/sync_api/_core/dendrite_element.py @@ -0,0 +1,239 @@ +from __future__ import annotations +import time +import base64 +import functools +import time +from typing import TYPE_CHECKING, Optional +from loguru import logger +from playwright.sync_api import Locator +from dendrite.browser._common._exceptions.dendrite_exception import ( + IncorrectOutcomeError, +) +from dendrite.logic.interfaces import SyncProtocol + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser.sync_api._core._managers.navigation_tracker import ( + NavigationTracker, +) +from dendrite.browser.sync_api._core._type_spec import Interaction +from dendrite.models.dto.make_interaction_dto import VerifyActionDTO +from dendrite.models.response.interaction_response import InteractionResponse + + +def perform_action(interaction_type: Interaction): + """ + Decorator for performing actions on DendriteElements. + + This decorator wraps methods of Element to handle interactions, + expected outcomes, and error handling. + + Args: + interaction_type (Interaction): The type of interaction being performed. + + Returns: + function: The decorated function. + """ + + def decorator(func): + + @functools.wraps(func) + def wrapper(self: Element, *args, **kwargs) -> InteractionResponse: + expected_outcome: Optional[str] = kwargs.pop("expected_outcome", None) + if not expected_outcome: + func(self, *args, **kwargs) + return InteractionResponse(status="success", message="") + page_before = self._dendrite_browser.get_active_page() + page_before_info = page_before.get_page_information() + soup = page_before._get_previous_soup() + screenshot_before = page_before_info.screenshot_base64 + tag_name = soup.find(attrs={"d-id": self.dendrite_id}) + func(self, *args, expected_outcome=expected_outcome, **kwargs) + self._wait_for_page_changes(page_before.url) + page_after = self._dendrite_browser.get_active_page() + screenshot_after = page_after.screenshot_manager.take_full_page_screenshot() + dto = VerifyActionDTO( + url=page_before.url, + dendrite_id=self.dendrite_id, + interaction_type=interaction_type, + expected_outcome=expected_outcome, + screenshot_before=screenshot_before, + screenshot_after=screenshot_after, + tag_name=str(tag_name), + ) + res = self._browser_api_client.verify_action(dto) + if res.status == "failed": + raise IncorrectOutcomeError( + message=res.message, screenshot_base64=screenshot_after + ) + return res + + return wrapper + + return decorator + + +class Element: + """ + Represents an element in the Dendrite browser environment. Wraps a Playwright Locator. + + This class provides methods for interacting with and manipulating + elements in the browser. + """ + + def __init__( + self, + dendrite_id: str, + locator: Locator, + dendrite_browser: Dendrite, + browser_api_client: SyncProtocol, + ): + """ + Initialize a Element. + + Args: + dendrite_id (str): The dendrite_id identifier for this element. + locator (Locator): The Playwright locator for this element. + dendrite_browser (Dendrite): The browser instance. + """ + self.dendrite_id = dendrite_id + self.locator = locator + self._dendrite_browser = dendrite_browser + self._browser_api_client = browser_api_client + + def outer_html(self): + return self.locator.evaluate("(element) => element.outerHTML") + + def screenshot(self) -> str: + """ + Take a screenshot of the element and return it as a base64-encoded string. + + Returns: + str: A base64-encoded string of the JPEG image. + Returns an empty string if the screenshot fails. + """ + image_data = self.locator.screenshot(type="jpeg", timeout=20000) + if image_data is None: + return "" + return base64.b64encode(image_data).decode() + + @perform_action("click") + def click( + self, + expected_outcome: Optional[str] = None, + wait_for_navigation: bool = True, + *args, + **kwargs, + ) -> InteractionResponse: + """ + Click the element. + + Args: + expected_outcome (Optional[str]): The expected outcome of the click action. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + InteractionResponse: The response from the interaction. + """ + timeout = kwargs.pop("timeout", 2000) + force = kwargs.pop("force", False) + page = self._dendrite_browser.get_active_page() + navigation_tracker = NavigationTracker(page) + navigation_tracker.start_nav_tracking() + try: + self.locator.click(*args, timeout=timeout, force=force, **kwargs) + except Exception as e: + try: + self.locator.click(*args, timeout=2000, force=True, **kwargs) + except Exception as e: + self.locator.dispatch_event("click", timeout=2000) + if wait_for_navigation: + has_navigated = navigation_tracker.has_navigated_since_start() + if has_navigated: + try: + start_time = time.time() + page.playwright_page.wait_for_load_state("load", timeout=2000) + wait_duration = time.time() - start_time + except Exception as e: + pass + return InteractionResponse(status="success", message="") + + @perform_action("fill") + def fill( + self, value: str, expected_outcome: Optional[str] = None, *args, **kwargs + ) -> InteractionResponse: + """ + Fill the element with a value. If the element itself is not fillable, + it attempts to find and fill a fillable child element. + + Args: + value (str): The value to fill the element with. + expected_outcome (Optional[str]): The expected outcome of the fill action. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + InteractionResponse: The response from the interaction. + """ + timeout = kwargs.pop("timeout", 2000) + try: + self.locator.fill(value, *args, timeout=timeout, **kwargs) + except Exception as e: + fillable_child = self.locator.locator( + 'input, textarea, [contenteditable="true"]' + ).first + fillable_child.fill(value, *args, timeout=timeout, **kwargs) + return InteractionResponse(status="success", message="") + + @perform_action("hover") + def hover( + self, expected_outcome: Optional[str] = None, *args, **kwargs + ) -> InteractionResponse: + """ + Hover over the element. + All additional arguments are passed to the Playwright fill method. + + Args: + expected_outcome (Optional[str]): The expected outcome of the hover action. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + InteractionResponse: The response from the interaction. + """ + timeout = kwargs.pop("timeout", 2000) + self.locator.hover(*args, timeout=timeout, **kwargs) + return InteractionResponse(status="success", message="") + + def focus(self): + """ + Focus on the element. + """ + self.locator.focus() + + def highlight(self): + """ + Highlights the element. This is a convenience method for debugging purposes. + """ + self.locator.highlight() + + def _wait_for_page_changes(self, old_url: str, timeout: float = 2000): + """ + Wait for page changes after an action. + + Args: + old_url (str): The URL before the action. + timeout (float): The maximum time (in milliseconds) to wait for changes. + + Returns: + bool: True if the page changed, False otherwise. + """ + timeout_in_seconds = timeout / 1000 + start_time = time.time() + while time.time() - start_time <= timeout_in_seconds: + page = self._dendrite_browser.get_active_page() + if page.url != old_url: + return True + time.sleep(0.1) + return False diff --git a/dendrite/browser/sync_api/_core/dendrite_page.py b/dendrite/browser/sync_api/_core/dendrite_page.py new file mode 100644 index 0000000..a1d7578 --- /dev/null +++ b/dendrite/browser/sync_api/_core/dendrite_page.py @@ -0,0 +1,377 @@ +import time +import pathlib +import re +import time +from typing import TYPE_CHECKING, Any, List, Literal, Optional, Sequence, Union +from bs4 import BeautifulSoup, Tag +from loguru import logger +from playwright.sync_api import Download, FilePayload, FrameLocator, Keyboard +from dendrite.browser.sync_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.sync_api._core.dendrite_element import Element +from dendrite.browser.sync_api._core.mixin.ask import AskMixin +from dendrite.browser.sync_api._core.mixin.click import ClickMixin +from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin +from dendrite.browser.sync_api._core.mixin.fill_fields import FillFieldsMixin +from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.sync_api._core.mixin.keyboard import KeyboardMixin +from dendrite.browser.sync_api._core.mixin.markdown import MarkdownMixin +from dendrite.browser.sync_api._core.mixin.wait_for import WaitForMixin +from dendrite.logic.interfaces import SyncProtocol +from dendrite.models.page_information import PageInformation + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core._managers.screenshot_manager import ( + ScreenshotManager, +) +from dendrite.browser.sync_api._core._utils import expand_iframes + + +class Page( + MarkdownMixin, + ExtractionMixin, + WaitForMixin, + AskMixin, + FillFieldsMixin, + ClickMixin, + KeyboardMixin, + GetElementMixin, +): + """ + Represents a page in the Dendrite browser environment. + + This class provides methods for interacting with and manipulating + pages in the browser. + """ + + def __init__( + self, + page: PlaywrightPage, + dendrite_browser: "Dendrite", + browser_api_client: SyncProtocol, + ): + self.playwright_page = page + self.screenshot_manager = ScreenshotManager(page) + self.dendrite_browser = dendrite_browser + self._browser_api_client = browser_api_client + self._last_main_frame_url = page.url + self._last_frame_navigated_timestamp = time.time() + self.playwright_page.on("framenavigated", self._on_frame_navigated) + + def _on_frame_navigated(self, frame): + if frame is self.playwright_page.main_frame: + self._last_main_frame_url = frame.url + self._last_frame_navigated_timestamp = time.time() + + @property + def url(self): + """ + Get the current URL of the page. + + Returns: + str: The current URL. + """ + return self.playwright_page.url + + @property + def keyboard(self) -> Keyboard: + """ + Get the keyboard object for the page. + + Returns: + Keyboard: The Playwright Keyboard object. + """ + return self.playwright_page.keyboard + + def _get_page(self) -> "Page": + return self + + def _get_dendrite_browser(self) -> "Dendrite": + return self.dendrite_browser + + def _get_logic_api(self) -> SyncProtocol: + return self._browser_api_client + + def goto( + self, + url: str, + timeout: Optional[float] = 30000, + wait_until: Optional[ + Literal["commit", "domcontentloaded", "load", "networkidle"] + ] = "load", + ) -> None: + """ + Navigate to a URL. + + Args: + url (str): The URL to navigate to. If no protocol is specified, 'https://' will be added. + timeout (Optional[float]): Maximum navigation time in milliseconds. + wait_until (Optional[Literal["commit", "domcontentloaded", "load", "networkidle"]]): + When to consider navigation succeeded. + """ + if not re.match("^\\w+://", url): + url = f"https://{url}" + self.playwright_page.goto(url, timeout=timeout, wait_until=wait_until) + + def get_download(self, timeout: float = 30000) -> Download: + """ + Retrieves the download event associated with. + + Args: + timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the download to complete. Defaults to 30. + + Returns: + The downloaded file data. + """ + return self.dendrite_browser._get_download(self.playwright_page, timeout) + + def _get_context(self, element: Any) -> Union[PlaywrightPage, FrameLocator]: + """ + Gets the correct context to be able to interact with an element on a different frame. + + e.g. if the element is inside an iframe, + the context will be the frame locator for that iframe. + + Args: + element (Any): The element to get the context for. + + Returns: + Union[Page, FrameLocator]: The context for the element. + """ + context = self.playwright_page + if isinstance(element, Tag): + full_path = element.get("iframe-path") + if full_path: + full_path = full_path[0] if isinstance(full_path, list) else full_path + for path in full_path.split("|"): + context = context.frame_locator(f"xpath=//iframe[@d-id='{path}']") + return context + + def scroll_to_bottom( + self, + timeout: float = 30000, + scroll_increment: int = 1000, + no_progress_limit: int = 3, + ) -> None: + """ + Scrolls to the bottom of the page until no more progress is made or a timeout occurs. + + Args: + timeout (float, optional): The maximum amount of time (in milliseconds) to continue scrolling. Defaults to 30000. + scroll_increment (int, optional): The number of pixels to scroll in each step. Defaults to 1000. + no_progress_limit (int, optional): The number of consecutive attempts with no progress before stopping. Defaults to 3. + + Returns: + None + """ + start_time = time.time() + last_scroll_position = 0 + no_progress_count = 0 + while True: + current_scroll_position = self.playwright_page.evaluate("window.scrollY") + scroll_height = self.playwright_page.evaluate("document.body.scrollHeight") + self.playwright_page.evaluate( + f"window.scrollTo(0, {current_scroll_position + scroll_increment})" + ) + if ( + self.playwright_page.viewport_size + and current_scroll_position + + self.playwright_page.viewport_size["height"] + >= scroll_height + ): + break + if current_scroll_position > last_scroll_position: + no_progress_count = 0 + else: + no_progress_count += 1 + if no_progress_count >= no_progress_limit: + break + if time.time() - start_time > timeout * 0.001: + break + last_scroll_position = current_scroll_position + time.sleep(0.1) + + def close(self) -> None: + """ + Closes the current page. + + Returns: + None + """ + self.playwright_page.close() + + def get_page_information(self, include_screenshot: bool = True) -> PageInformation: + """ + Retrieves information about the current page, including the URL, raw HTML, and a screenshot. + + Returns: + PageInformation: An object containing the page's URL, raw HTML, and a screenshot in base64 format. + """ + if include_screenshot: + base64 = self.screenshot_manager.take_full_page_screenshot() + else: + base64 = "No screenshot available" + soup = self._get_soup() + return PageInformation( + url=self.playwright_page.url, + raw_html=str(soup), + screenshot_base64=base64, + time_since_frame_navigated=self.get_time_since_last_frame_navigated(), + ) + + def _generate_dendrite_ids(self): + """ + Attempts to generate Dendrite IDs in the DOM by executing a script. + + This method will attempt to generate the Dendrite IDs up to 3 times. If all attempts fail, + an exception is raised. + + Raises: + Exception: If the Dendrite IDs could not be generated after 3 attempts. + """ + tries = 0 + while tries < 3: + try: + self.playwright_page.evaluate(GENERATE_DENDRITE_IDS_SCRIPT) + return + except Exception as e: + self.playwright_page.wait_for_load_state(state="load", timeout=3000) + logger.exception( + f"Failed to generate dendrite IDs: {e}, attempt {tries + 1}/3" + ) + tries += 1 + raise DendriteException("Failed to add d-ids to DOM.") + + def scroll_through_entire_page(self) -> None: + """ + Scrolls through the entire page. + + Returns: + None + """ + self.scroll_to_bottom() + + def upload_files( + self, + files: Union[ + str, + pathlib.Path, + FilePayload, + Sequence[Union[str, pathlib.Path]], + Sequence[FilePayload], + ], + timeout: float = 30000, + ) -> None: + """ + Uploads files to the page using a file chooser. + + Args: + files (Union[str, pathlib.Path, FilePayload, Sequence[Union[str, pathlib.Path]], Sequence[FilePayload]]): The file(s) to be uploaded. + This can be a file path, a `FilePayload` object, or a sequence of file paths or `FilePayload` objects. + timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the file chooser to be ready. Defaults to 30. + + Returns: + None + """ + file_chooser = self.dendrite_browser._get_filechooser( + self.playwright_page, timeout + ) + file_chooser.set_files(files) + + def get_content(self): + """ + Retrieves the content of the current page. + + Returns: + str: The HTML content of the current page. + """ + return self.playwright_page.content() + + def _get_soup(self) -> BeautifulSoup: + """ + Retrieves the page source as a BeautifulSoup object, with an option to exclude hidden elements. + Generates Dendrite IDs in the DOM and expands iframes. + + Returns: + BeautifulSoup: The parsed HTML of the current page. + """ + self._generate_dendrite_ids() + page_source = self.playwright_page.content() + soup = BeautifulSoup(page_source, "lxml") + self._expand_iframes(soup) + self._previous_soup = soup + return soup + + def _get_previous_soup(self) -> BeautifulSoup: + """ + Retrieves the page source generated by the latest _get_soup() call as a Beautiful soup object. If it hasn't been called yet, it will call it. + """ + if self._previous_soup is None: + return self._get_soup() + return self._previous_soup + + def _expand_iframes(self, page_source: BeautifulSoup): + """ + Expands iframes in the given page source to make their content accessible. + + Args: + page_source (BeautifulSoup): The parsed HTML content of the page. + + Returns: + None + """ + expand_iframes(self.playwright_page, page_source) + + def _get_all_elements_from_selector(self, selector: str) -> List[Element]: + dendrite_elements: List[Element] = [] + soup = self._get_soup() + elements = soup.select(selector) + for element in elements: + frame = self._get_context(element) + d_id = element.get("d-id", "") + locator = frame.locator(f"xpath=//*[@d-id='{d_id}']") + if not d_id: + continue + if isinstance(d_id, list): + d_id = d_id[0] + dendrite_elements.append( + Element(d_id, locator, self.dendrite_browser, self._browser_api_client) + ) + return dendrite_elements + + def _dump_html(self, path: str) -> None: + """ + Saves the current page's HTML content to a file. + + Args: + path (str): The file path where the HTML content should be saved. + + Returns: + None + """ + with open(path, "w") as f: + f.write(self.playwright_page.content()) + + def get_time_since_last_frame_navigated(self) -> float: + """ + Get the time elapsed since the last URL change. + + Returns: + float: The number of seconds elapsed since the last URL change. + """ + return time.time() - self._last_frame_navigated_timestamp + + def check_if_renavigated(self, initial_url: str, wait_time: float = 0.1) -> bool: + """ + Waits for a short period and checks if a main frame navigation has occurred. + + Args: + wait_time (float): The time to wait in seconds. Defaults to 0.1 seconds. + + Returns: + bool: True if a main frame navigation occurred, False otherwise. + """ + time.sleep(wait_time) + return self._last_main_frame_url != initial_url diff --git a/dendrite/browser/sync_api/_core/event_sync.py b/dendrite/browser/sync_api/_core/event_sync.py new file mode 100644 index 0000000..4351eee --- /dev/null +++ b/dendrite/browser/sync_api/_core/event_sync.py @@ -0,0 +1,45 @@ +import time +import time +from typing import Generic, Optional, Type, TypeVar +from playwright.sync_api import Download, FileChooser, Page + +Events = TypeVar("Events", Download, FileChooser) +mapping = {Download: "download", FileChooser: "filechooser"} + + +class EventSync(Generic[Events]): + + def __init__(self, event_type: Type[Events]): + self.event_type = event_type + self.event_set = False + self.data: Optional[Events] = None + + def get_data(self, pw_page: Page, timeout: float = 30000) -> Events: + start_time = time.time() + while not self.event_set: + elapsed_time = (time.time() - start_time) * 1000 + if elapsed_time > timeout: + raise TimeoutError(f'Timeout waiting for event "{self.event_type}".') + pw_page.wait_for_timeout(0) + time.sleep(0.01) + data = self.data + self.data = None + self.event_set = False + if data is None: + raise ValueError("Data is None for event type: ", self.event_type) + return data + + def set_event(self, data: Events) -> None: + """ + Sets the event and stores the provided data. + + This method is used to signal that the data is ready to be retrieved by any waiting tasks. + + Args: + data (T): The data to be stored and associated with the event. + + Returns: + None + """ + self.data = data + self.event_set = True diff --git a/dendrite/browser/sync_api/_core/mixin/__init__.py b/dendrite/browser/sync_api/_core/mixin/__init__.py new file mode 100644 index 0000000..046a61c --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/__init__.py @@ -0,0 +1,21 @@ +from .ask import AskMixin +from .click import ClickMixin +from .extract import ExtractionMixin +from .fill_fields import FillFieldsMixin +from .get_element import GetElementMixin +from .keyboard import KeyboardMixin +from .markdown import MarkdownMixin +from .screenshot import ScreenshotMixin +from .wait_for import WaitForMixin + +__all__ = [ + "AskMixin", + "ClickMixin", + "ExtractionMixin", + "FillFieldsMixin", + "GetElementMixin", + "KeyboardMixin", + "MarkdownMixin", + "ScreenshotMixin", + "WaitForMixin", +] diff --git a/dendrite/browser/sync_api/_core/mixin/ask.py b/dendrite/browser/sync_api/_core/mixin/ask.py new file mode 100644 index 0000000..ea7aea4 --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/ask.py @@ -0,0 +1,189 @@ +import time +import time +from typing import Optional, Type, overload +from loguru import logger +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core._type_spec import ( + JsonSchema, + PydanticModel, + TypeSpec, + convert_to_type_spec, + to_json_schema, +) +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.models.dto.ask_page_dto import AskPageDTO + +TIMEOUT_INTERVAL = [150, 450, 1000] + + +class AskMixin(DendritePageProtocol): + + @overload + def ask(self, prompt: str, type_spec: Type[str]) -> str: + """ + Asks a question about the current page and expects a response of type `str`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[str]): The expected return type, which is `str`. + + Returns: + AskPageResponse[str]: The response object containing the result of type `str`. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[bool]) -> bool: + """ + Asks a question about the current page and expects a responseof type `bool`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[bool]): The expected return type, which is `bool`. + + Returns: + AskPageResponse[bool]: The response object containing the result of type `bool`. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[int]) -> int: + """ + Asks a question about the current page and expects a response of type `int`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[int]): The expected return type, which is `int`. + + Returns: + AskPageResponse[int]: The response object containing the result of type `int`. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[float]) -> float: + """ + Asks a question about the current page and expects a response of type `float`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[float]): The expected return type, which is `float`. + + Returns: + AskPageResponse[float]: The response object containing the result of type `float`. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[PydanticModel]) -> PydanticModel: + """ + Asks a question about the current page and expects a response of a custom `PydanticModel`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[PydanticModel]): The expected return type, which is a `PydanticModel`. + + Returns: + AskPageResponse[PydanticModel]: The response object containing the result of the specified Pydantic model type. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[JsonSchema]) -> JsonSchema: + """ + Asks a question about the current page and expects a response conforming to a `JsonSchema`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[JsonSchema]): The expected return type, which is a `JsonSchema`. + + Returns: + AskPageResponse[JsonSchema]: The response object containing the result conforming to the specified JSON schema. + """ + + @overload + def ask(self, prompt: str, type_spec: None = None) -> JsonSchema: + """ + Asks a question without specifying a type and expects a response conforming to a default `JsonSchema`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (None, optional): The expected return type, which is `None` by default. + + Returns: + AskPageResponse[JsonSchema]: The response object containing the result conforming to the default JSON schema. + """ + + def ask( + self, prompt: str, type_spec: Optional[TypeSpec] = None, timeout: int = 15000 + ) -> TypeSpec: + """ + Asks a question and processes the response based on the specified type. + + This method sends a request to ask a question with the specified prompt and processes the response. + If a type specification is provided, the response is converted to the specified type. In case of failure, + a DendriteException is raised with relevant details. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Optional[TypeSpec], optional): The expected return type, which can be a type or a schema. Defaults to None. + + Returns: + AskPageResponse[Any]: The response object containing the result, converted to the specified type if provided. + + Raises: + DendriteException: If the request fails, the exception includes the failure message and a screenshot. + """ + start_time = time.time() + attempt_start = start_time + attempt = -1 + while True: + attempt += 1 + current_timeout = ( + TIMEOUT_INTERVAL[attempt] + if len(TIMEOUT_INTERVAL) > attempt + else TIMEOUT_INTERVAL[-1] * 1.75 + ) + elapsed_time = time.time() - start_time + remaining_time = timeout * 0.001 - elapsed_time + if remaining_time <= 0: + logger.warning( + f"Timeout reached for '{prompt}' after {attempt + 1} attempts" + ) + break + prev_attempt_time = time.time() - attempt_start + sleep_time = min( + max(current_timeout * 0.001 - prev_attempt_time, 0), remaining_time + ) + logger.debug(f"Waiting for {sleep_time} seconds before retrying") + time.sleep(sleep_time) + attempt_start = time.time() + logger.info(f"Asking '{prompt}' | Attempt {attempt + 1}") + page = self._get_page() + page_information = page.get_page_information() + schema = to_json_schema(type_spec) if type_spec else None + if elapsed_time < 5: + time_prompt = f"This page was loaded {elapsed_time} seconds ago, so it might still be loading. If the page is still loading, return failed status." + else: + time_prompt = "" + entire_prompt = prompt + time_prompt + dto = AskPageDTO( + page_information=page_information, + prompt=entire_prompt, + return_schema=schema, + ) + try: + res = self._get_logic_api().ask_page(dto) + logger.debug(f"Got response in {time.time() - attempt_start} seconds") + if res.status == "error": + logger.warning( + f"Error response on attempt {attempt + 1}: {res.return_data}" + ) + continue + converted_res = res.return_data + if type_spec is not None: + converted_res = convert_to_type_spec(type_spec, res.return_data) + return converted_res + except Exception as e: + logger.error(f"Exception occurred on attempt {attempt + 1}: {str(e)}") + if attempt == len(TIMEOUT_INTERVAL) - 1: + raise + raise DendriteException( + message=f"Failed to get response for '{prompt}' after {attempt + 1} attempts", + screenshot_base64=page_information.screenshot_base64, + ) diff --git a/dendrite/browser/sync_api/_core/mixin/click.py b/dendrite/browser/sync_api/_core/mixin/click.py new file mode 100644 index 0000000..0b6adaa --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/click.py @@ -0,0 +1,56 @@ +from typing import Optional +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.models.response.interaction_response import InteractionResponse + + +class ClickMixin(GetElementMixin, DendritePageProtocol): + + def click( + self, + prompt: str, + expected_outcome: Optional[str] = None, + use_cache: bool = True, + timeout: int = 15000, + force: bool = False, + *args, + **kwargs, + ) -> InteractionResponse: + """ + Clicks an element on the page based on the provided prompt. + + This method combines the functionality of get_element and click, + allowing for a more concise way to interact with elements on the page. + + Args: + prompt (str): The prompt describing the element to be clicked. + expected_outcome (Optional[str]): The expected outcome of the click action. + use_cache (bool, optional): Whether to use cached results for element retrieval. Defaults to True. + timeout (int, optional): The timeout (in milliseconds) for the click operation. Defaults to 15000. + force (bool, optional): Whether to force the click operation. Defaults to False. + *args: Additional positional arguments for the click operation. + **kwargs: Additional keyword arguments for the click operation. + + Returns: + InteractionResponse: The response from the interaction. + + Raises: + DendriteException: If no suitable element is found or if the click operation fails. + """ + augmented_prompt = prompt + "\n\nThe element should be clickable." + element = self.get_element( + augmented_prompt, use_cache=use_cache, timeout=timeout + ) + if not element: + raise DendriteException( + message=f"No element found with the prompt: {prompt}", + screenshot_base64="", + ) + return element.click( + *args, + expected_outcome=expected_outcome, + timeout=timeout, + force=force, + **kwargs, + ) diff --git a/dendrite/browser/sync_api/_core/mixin/extract.py b/dendrite/browser/sync_api/_core/mixin/extract.py new file mode 100644 index 0000000..b6aa5ad --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/extract.py @@ -0,0 +1,211 @@ +import time +import time +from typing import Any, List, Optional, Type, overload +from loguru import logger +from dendrite.browser.sync_api._core._managers.navigation_tracker import ( + NavigationTracker, +) +from dendrite.browser.sync_api._core._type_spec import ( + JsonSchema, + PydanticModel, + TypeSpec, + convert_to_type_spec, + to_json_schema, +) +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.response.extract_response import ExtractResponse + +CACHE_TIMEOUT = 5 + + +class ExtractionMixin(DendritePageProtocol): + """ + Mixin that provides extraction functionality for web pages. + + This mixin provides various `extract` methods that allow extracting + different types of data (e.g., bool, int, float, string, Pydantic models, etc.) + from a web page based on a given prompt. + """ + + @overload + def extract( + self, + prompt: str, + type_spec: Type[bool], + use_cache: bool = True, + timeout: int = 180, + ) -> bool: ... + + @overload + def extract( + self, + prompt: str, + type_spec: Type[int], + use_cache: bool = True, + timeout: int = 180, + ) -> int: ... + + @overload + def extract( + self, + prompt: str, + type_spec: Type[float], + use_cache: bool = True, + timeout: int = 180, + ) -> float: ... + + @overload + def extract( + self, + prompt: str, + type_spec: Type[str], + use_cache: bool = True, + timeout: int = 180, + ) -> str: ... + + @overload + def extract( + self, + prompt: Optional[str], + type_spec: Type[PydanticModel], + use_cache: bool = True, + timeout: int = 180, + ) -> PydanticModel: ... + + @overload + def extract( + self, + prompt: Optional[str], + type_spec: JsonSchema, + use_cache: bool = True, + timeout: int = 180, + ) -> JsonSchema: ... + + @overload + def extract( + self, + prompt: str, + type_spec: None = None, + use_cache: bool = True, + timeout: int = 180, + ) -> Any: ... + + def extract( + self, + prompt: Optional[str], + type_spec: Optional[TypeSpec] = None, + use_cache: bool = True, + timeout: int = 180, + ) -> TypeSpec: + """ + Extract data from a web page based on a prompt and optional type specification. + Args: + prompt (Optional[str]): The prompt to describe the information to extract. + type_spec (Optional[TypeSpec], optional): The type specification for the extracted data. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached scripts before falling back to the + extraction agent for the remaining time that will attempt to generate a new script. Defaults to 15000 (15 seconds). + + Returns: + ExtractResponse: The extracted data wrapped in a ExtractResponse object. + Raises: + TimeoutError: If the extraction process exceeds the specified timeout. + """ + logger.info(f"Starting extraction with prompt: {prompt}") + json_schema = None + if type_spec: + json_schema = to_json_schema(type_spec) + logger.debug(f"Type specification converted to JSON schema: {json_schema}") + if prompt is None: + prompt = "" + start_time = time.time() + page = self._get_page() + navigation_tracker = NavigationTracker(page) + navigation_tracker.start_nav_tracking() + if use_cache: + logger.info("Cache available, attempting to use cached extraction") + result = attempt_extraction_with_backoff( + self, + prompt, + json_schema, + remaining_timeout=CACHE_TIMEOUT, + only_use_cache=True, + ) + if result: + return convert_and_return_result(result, type_spec) + logger.info( + "Using extraction agent to perform extraction, since no cache was found or failed." + ) + result = attempt_extraction_with_backoff( + self, + prompt, + json_schema, + remaining_timeout=timeout - (time.time() - start_time), + only_use_cache=False, + ) + if result: + return convert_and_return_result(result, type_spec) + logger.error(f"Extraction failed after {time.time() - start_time:.2f} seconds") + return None + + +def attempt_extraction_with_backoff( + obj: DendritePageProtocol, + prompt: str, + json_schema: Optional[JsonSchema], + remaining_timeout: float = 180.0, + only_use_cache: bool = False, +) -> Optional[ExtractResponse]: + TIMEOUT_INTERVAL: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0] + total_elapsed_time = 0 + start_time = time.time() + for current_timeout in TIMEOUT_INTERVAL: + if total_elapsed_time >= remaining_timeout: + logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") + return None + request_start_time = time.time() + page = obj._get_page() + page_information = page.get_page_information( + include_screenshot=not only_use_cache + ) + extract_dto = ExtractDTO( + page_information=page_information, + prompt=prompt, + return_data_json_schema=json_schema, + use_screenshot=True, + use_cache=only_use_cache, + force_use_cache=only_use_cache, + ) + res: ExtractResponse = obj._get_logic_api().extract(extract_dto) + request_duration = time.time() - request_start_time + if res.status == "impossible": + logger.error(f"Impossible to extract data. Reason: {res.message}") + return None + if res.status == "success": + logger.success( + f"Extraction successful: '{res.message}'\nUsed cache: {res.used_cache}" + ) + return res + sleep_duration = max(0, current_timeout - request_duration) + logger.info( + f"Extraction attempt failed. Status: {res.status}\nMessage: {res.message}\nSleeping for {sleep_duration:.2f} seconds" + ) + time.sleep(sleep_duration) + total_elapsed_time = time.time() - start_time + logger.error( + f"All extraction attempts failed after {total_elapsed_time:.2f} seconds" + ) + return None + + +def convert_and_return_result( + res: ExtractResponse, type_spec: Optional[TypeSpec] +) -> TypeSpec: + converted_res = res.return_data + if type_spec is not None: + logger.debug("Converting extraction result to specified type") + converted_res = convert_to_type_spec(type_spec, res.return_data) + logger.info("Extraction process completed successfully") + return converted_res diff --git a/dendrite/browser/sync_api/_core/mixin/fill_fields.py b/dendrite/browser/sync_api/_core/mixin/fill_fields.py new file mode 100644 index 0000000..e49825b --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/fill_fields.py @@ -0,0 +1,76 @@ +import time +from typing import Any, Dict, Optional +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.models.response.interaction_response import InteractionResponse + + +class FillFieldsMixin(GetElementMixin, DendritePageProtocol): + + def fill_fields(self, fields: Dict[str, Any]): + """ + Fills multiple fields on the page with the provided values. + + This method iterates through the given dictionary of fields and their corresponding values, + making a separate fill request for each key-value pair. + + Args: + fields (Dict[str, Any]): A dictionary where each key is a field identifier (e.g., a prompt or selector) + and each value is the content to fill in that field. + + Returns: + None + + Note: + This method will make multiple fill requests, one for each key in the 'fields' dictionary. + """ + for field, value in fields.items(): + prompt = f"I'll be filling in text in several fields with these keys: {fields.keys()} in this page. Get the field best described as '{field}'. I want to fill it with a '{type(value)}' type value." + self.fill(prompt, value) + time.sleep(0.5) + + def fill( + self, + prompt: str, + value: str, + expected_outcome: Optional[str] = None, + use_cache: bool = True, + timeout: int = 15000, + *args, + kwargs={}, + ) -> InteractionResponse: + """ + Fills an element on the page with the provided value based on the given prompt. + + This method combines the functionality of get_element and fill, + allowing for a more concise way to interact with elements on the page. + + Args: + prompt (str): The prompt describing the element to be filled. + value (str): The value to fill the element with. + expected_outcome (Optional[str]): The expected outcome of the fill action. + use_cache (bool, optional): Whether to use cached results for element retrieval. Defaults to True. + max_retries (int, optional): The maximum number of retry attempts for element retrieval. Defaults to 3. + timeout (int, optional): The timeout (in milliseconds) for the fill operation. Defaults to 15000. + *args: Additional positional arguments for the fill operation. + kwargs: Additional keyword arguments for the fill operation. + + Returns: + InteractionResponse: The response from the interaction. + + Raises: + DendriteException: If no suitable element is found or if the fill operation fails. + """ + augmented_prompt = prompt + "\n\nMake sure the element can be filled with text." + element = self.get_element( + augmented_prompt, use_cache=use_cache, timeout=timeout + ) + if not element: + raise DendriteException( + message=f"No element found with the prompt: {prompt}", + screenshot_base64="", + ) + return element.fill( + value, *args, expected_outcome=expected_outcome, timeout=timeout, **kwargs + ) diff --git a/dendrite/browser/sync_api/_core/mixin/get_element.py b/dendrite/browser/sync_api/_core/mixin/get_element.py new file mode 100644 index 0000000..1d4694b --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/get_element.py @@ -0,0 +1,286 @@ +import time +import time +from typing import Dict, List, Literal, Optional, Union, overload +from loguru import logger +from dendrite.browser.sync_api._core._utils import get_elements_from_selectors_soup +from dendrite.browser.sync_api._core.dendrite_element import Element +from dendrite.browser.sync_api._core.models.response import ElementsResponse +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.models.dto.get_elements_dto import CheckSelectorCacheDTO, GetElementsDTO +from dendrite.models.response.get_element_response import GetElementResponse + +CACHE_TIMEOUT = 5 + + +class GetElementMixin(DendritePageProtocol): + + @overload + def get_elements( + self, + prompt_or_elements: str, + use_cache: bool = True, + timeout: int = 15000, + context: str = "", + ) -> List[Element]: + """ + Retrieves a list of Dendrite elements based on a string prompt. + + Args: + prompt_or_elements (str): The prompt describing the elements to be retrieved. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + context (str, optional): Additional context for the retrieval. Defaults to an empty string. + + Returns: + List[Element]: A list of Dendrite elements found on the page. + """ + + @overload + def get_elements( + self, + prompt_or_elements: Dict[str, str], + use_cache: bool = True, + timeout: int = 15000, + context: str = "", + ) -> ElementsResponse: + """ + Retrieves Dendrite elements based on a dictionary. + + Args: + prompt_or_elements (Dict[str, str]): A dictionary where keys are field names and values are prompts describing the elements to be retrieved. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + context (str, optional): Additional context for the retrieval. Defaults to an empty string. + + Returns: + ElementsResponse: A response object containing the retrieved elements with attributes matching the keys in the dict. + """ + + def get_elements( + self, + prompt_or_elements: Union[str, Dict[str, str]], + use_cache: bool = True, + timeout: int = 15000, + context: str = "", + ) -> Union[List[Element], ElementsResponse]: + """ + Retrieves Dendrite elements based on either a string prompt or a dictionary of prompts. + + This method determines the type of the input (string or dictionary) and retrieves the appropriate elements. + If the input is a string, it fetches a list of elements. If the input is a dictionary, it fetches elements for each key-value pair. + + Args: + prompt_or_elements (Union[str, Dict[str, str]]): The prompt or dictionary of prompts for element retrieval. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + context (str, optional): Additional context for the retrieval. Defaults to an empty string. + + Returns: + Union[List[Element], ElementsResponse]: A list of elements or a response object containing the retrieved elements. + + Raises: + ValueError: If the input is neither a string nor a dictionary. + """ + return self._get_element( + prompt_or_elements, + only_one=False, + use_cache=use_cache, + timeout=timeout / 1000, + ) + + def get_element( + self, prompt: str, use_cache=True, timeout=15000 + ) -> Optional[Element]: + """ + Retrieves a single Dendrite element based on the provided prompt. + + Args: + prompt (str): The prompt describing the element to be retrieved. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + + Returns: + Element: The retrieved element. + """ + return self._get_element( + prompt, only_one=True, use_cache=use_cache, timeout=timeout / 1000 + ) + + @overload + def _get_element( + self, prompt_or_elements: str, only_one: Literal[True], use_cache: bool, timeout + ) -> Optional[Element]: + """ + Retrieves a single Dendrite element based on the provided prompt. + + Args: + prompt (Union[str, Dict[str, str]]): The prompt describing the element to be retrieved. + only_one (Literal[True]): Indicates that only one element should be retrieved. + use_cache (bool): Whether to use cached results. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + + Returns: + Element: The retrieved element. + """ + + @overload + def _get_element( + self, + prompt_or_elements: Union[str, Dict[str, str]], + only_one: Literal[False], + use_cache: bool, + timeout, + ) -> Union[List[Element], ElementsResponse]: + """ + Retrieves a list of Dendrite elements based on the provided prompt. + + Args: + prompt (str): The prompt describing the elements to be retrieved. + only_one (Literal[False]): Indicates that multiple elements should be retrieved. + use_cache (bool): Whether to use cached results. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + + Returns: + List[Element]: A list of retrieved elements. + """ + + def _get_element( + self, + prompt_or_elements: Union[str, Dict[str, str]], + only_one: bool, + use_cache: bool, + timeout: float, + ) -> Union[Optional[Element], List[Element], ElementsResponse]: + """ + Retrieves Dendrite elements based on the provided prompt, either a single element or a list of elements. + + This method sends a request with the prompt and retrieves the elements based on the `only_one` flag. + + Args: + prompt_or_elements (Union[str, Dict[str, str]]): The prompt or dictionary of prompts for element retrieval. + only_one (bool): Whether to retrieve only one element or a list of elements. + use_cache (bool): Whether to use cached results. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + + Returns: + Union[Element, List[Element], ElementsResponse]: The retrieved element, list of elements, or response object. + """ + start_time = time.time() + page = self._get_page() + if use_cache == True: + logger.debug("Attempting to use cached selectors") + res = attempt_with_backoff( + self, + prompt_or_elements, + only_one, + remaining_timeout=CACHE_TIMEOUT, + only_use_cache=True, + ) + if res: + return res + else: + logger.debug( + f"After attempting to use cached selectors several times without success, let's find the elements using the find element agent." + ) + logger.info( + "Proceeding to use the find element agent to find the requested elements." + ) + res = attempt_with_backoff( + self, + prompt_or_elements, + only_one, + remaining_timeout=timeout - (time.time() - start_time), + only_use_cache=False, + ) + if res: + return res + logger.error( + f"Failed to retrieve elements within the specified timeout of {timeout} seconds" + ) + return None + + +def attempt_with_backoff( + obj: DendritePageProtocol, + prompt_or_elements: Union[str, Dict[str, str]], + only_one: bool, + remaining_timeout: float, + only_use_cache: bool = False, +) -> Union[Optional[Element], List[Element], ElementsResponse]: + TIMEOUT_INTERVAL: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0] + total_elapsed_time = 0 + start_time = time.time() + for current_timeout in TIMEOUT_INTERVAL: + if total_elapsed_time >= remaining_timeout: + logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") + return None + request_start_time = time.time() + page = obj._get_page() + page_information = page.get_page_information( + include_screenshot=not only_use_cache + ) + dto = GetElementsDTO( + page_information=page_information, + prompt=prompt_or_elements, + use_cache=only_use_cache, + only_one=only_one, + force_use_cache=only_use_cache, + ) + res = obj._get_logic_api().get_element(dto) + request_duration = time.time() - request_start_time + if res.status == "impossible": + logger.error( + f"Impossible to get elements for '{prompt_or_elements}'. Reason: {res.message}" + ) + return None + if res.status == "success": + logger.success(f"d[id]: {res.d_id} Selectors:{res.selectors}") + response = get_elements_from_selectors_soup( + page, page._get_previous_soup(), res, only_one + ) + if response: + return response + sleep_duration = max(0, current_timeout - request_duration) + logger.info( + f"Failed to get elements for prompt:\n\n'{prompt_or_elements}'\n\nStatus: {res.status}\n\nMessage: {res.message}\n\nSleeping for {sleep_duration:.2f} seconds" + ) + time.sleep(sleep_duration) + total_elapsed_time = time.time() - start_time + logger.error(f"All attempts failed after {total_elapsed_time:.2f} seconds") + return None + + +def get_elements_from_selectors( + obj: DendritePageProtocol, res: GetElementResponse, only_one: bool +) -> Union[Optional[Element], List[Element], ElementsResponse]: + if isinstance(res.selectors, dict): + result = {} + for key, selectors in res.selectors.items(): + for selector in selectors: + page = obj._get_page() + dendrite_elements = page._get_all_elements_from_selector(selector) + if len(dendrite_elements) > 0: + result[key] = dendrite_elements[0] + break + return ElementsResponse(result) + elif isinstance(res.selectors, list): + for selector in reversed(res.selectors): + page = obj._get_page() + dendrite_elements = page._get_all_elements_from_selector(selector) + if len(dendrite_elements) > 0: + return dendrite_elements[0] if only_one else dendrite_elements + return None diff --git a/dendrite/browser/sync_api/_core/mixin/keyboard.py b/dendrite/browser/sync_api/_core/mixin/keyboard.py new file mode 100644 index 0000000..1ab7894 --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/keyboard.py @@ -0,0 +1,62 @@ +from typing import Literal, Union +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol + + +class KeyboardMixin(DendritePageProtocol): + + def press( + self, + key: Union[ + str, + Literal[ + "Enter", + "Tab", + "Escape", + "Backspace", + "ArrowUp", + "ArrowDown", + "ArrowLeft", + "ArrowRight", + ], + ], + hold_shift: bool = False, + hold_ctrl: bool = False, + hold_alt: bool = False, + hold_cmd: bool = False, + ): + """ + Presses a keyboard key on the active page, optionally with modifier keys. + + Args: + key (Union[str, Literal["Enter", "Tab", "Escape", "Backspace", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight"]]): The main key to be pressed. + hold_shift (bool, optional): Whether to hold the Shift key. Defaults to False. + hold_ctrl (bool, optional): Whether to hold the Control key. Defaults to False. + hold_alt (bool, optional): Whether to hold the Alt key. Defaults to False. + hold_cmd (bool, optional): Whether to hold the Command key (Meta on some systems). Defaults to False. + + Returns: + Any: The result of the key press operation. + + Raises: + DendriteException: If the key press operation fails. + """ + modifiers = [] + if hold_shift: + modifiers.append("Shift") + if hold_ctrl: + modifiers.append("Control") + if hold_alt: + modifiers.append("Alt") + if hold_cmd: + modifiers.append("Meta") + if modifiers: + key = "+".join(modifiers + [key]) + try: + page = self._get_page() + page.keyboard.press(key) + except Exception as e: + raise DendriteException( + message=f"Failed to press key: {key}. Error: {str(e)}", + screenshot_base64="", + ) diff --git a/dendrite/browser/sync_api/_core/mixin/markdown.py b/dendrite/browser/sync_api/_core/mixin/markdown.py new file mode 100644 index 0000000..fce98d2 --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/markdown.py @@ -0,0 +1,23 @@ +import re +from typing import Optional +from bs4 import BeautifulSoup +from markdownify import markdownify as md +from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol + + +class MarkdownMixin(ExtractionMixin, DendritePageProtocol): + + def markdown(self, prompt: Optional[str] = None): + page = self._get_page() + page_information = page.get_page_information() + if prompt: + extract_prompt = f"Create a script that returns the HTML from one element from the DOM that best matches this requested section of the website.\n\nDescription of section: '{prompt}'\n\nWe will be converting your returned HTML to markdown, so just return ONE stringified HTML element and nothing else. It's OK if extra information is present. Example script: 'response_data = soup.find('tag', {{'attribute': 'value'}}).prettify()'" + res = self.extract(extract_prompt) + markdown_text = md(res) + cleaned_markdown = re.sub("\\n{3,}", "\n\n", markdown_text) + return cleaned_markdown + else: + markdown_text = md(page_information.raw_html) + cleaned_markdown = re.sub("\\n{3,}", "\n\n", markdown_text) + return cleaned_markdown diff --git a/dendrite/browser/sync_api/_core/mixin/screenshot.py b/dendrite/browser/sync_api/_core/mixin/screenshot.py new file mode 100644 index 0000000..bd3fab2 --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/screenshot.py @@ -0,0 +1,20 @@ +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol + + +class ScreenshotMixin(DendritePageProtocol): + + def screenshot(self, full_page: bool = False) -> str: + """ + Take a screenshot of the current page. + + Args: + full_page (bool, optional): If True, captures the full page. If False, captures only the viewport. Defaults to False. + + Returns: + str: A base64 encoded string of the screenshot in JPEG format. + """ + page = self._get_page() + if full_page: + return page.screenshot_manager.take_full_page_screenshot() + else: + return page.screenshot_manager.take_viewport_screenshot() diff --git a/dendrite/browser/sync_api/_core/mixin/wait_for.py b/dendrite/browser/sync_api/_core/mixin/wait_for.py new file mode 100644 index 0000000..df29d12 --- /dev/null +++ b/dendrite/browser/sync_api/_core/mixin/wait_for.py @@ -0,0 +1,53 @@ +import time +import time +from loguru import logger +from dendrite.browser._common._exceptions.dendrite_exception import ( + DendriteException, + PageConditionNotMet, +) +from dendrite.browser.sync_api._core.mixin.ask import AskMixin +from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol + + +class WaitForMixin(AskMixin, DendritePageProtocol): + + def wait_for(self, prompt: str, timeout: float = 30000): + """ + Waits for the condition specified in the prompt to become true by periodically checking the page content. + + This method attempts to retrieve the page information and evaluate whether the specified + condition (provided in the prompt) is met. It continues to retry until the total elapsed time + exceeds the specified timeout. + + Args: + prompt (str): The prompt to determine the condition to wait for on the page. + timeout (float, optional): The maximum time (in milliseconds) to wait for the condition. Defaults to 15000. + + Returns: + Any: The result of the condition evaluation if successful. + + Raises: + PageConditionNotMet: If the condition is not met within the specified timeout. + """ + start_time = time.time() + time.sleep(0.2) + while True: + elapsed_time = (time.time() - start_time) * 1000 + if elapsed_time >= timeout: + break + page = self._get_page() + page_information = page.get_page_information() + prompt_with_instruction = f"Prompt: '{prompt}'\n\nReturn a boolean that determines if the requested information or thing is available on the page. {round(page_information.time_since_frame_navigated, 2)} seconds have passed since the page first loaded." + try: + res = self.ask(prompt_with_instruction, bool) + if res: + return res + except DendriteException as e: + logger.debug(f"Attempt failed: {e.message}") + time.sleep(0.5) + page = self._get_page() + page_information = page.get_page_information() + raise PageConditionNotMet( + message=f"Failed to wait for the requested condition within the {timeout}ms timeout.", + screenshot_base64=page_information.screenshot_base64, + ) diff --git a/dendrite/browser/sync_api/_core/models/__init__.py b/dendrite/browser/sync_api/_core/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/_core/models/authentication.py b/dendrite/browser/sync_api/_core/models/authentication.py new file mode 100644 index 0000000..250a8ce --- /dev/null +++ b/dendrite/browser/sync_api/_core/models/authentication.py @@ -0,0 +1,47 @@ +from typing import List, Literal, Optional +from pydantic import BaseModel +from typing_extensions import TypedDict + + +class Cookie(TypedDict, total=False): + name: str + value: str + domain: str + path: str + expires: float + httpOnly: bool + secure: bool + sameSite: Literal["Lax", "None", "Strict"] + + +class LocalStorageEntry(TypedDict): + name: str + value: str + + +class OriginState(TypedDict): + origin: str + localStorage: List[LocalStorageEntry] + + +class StorageState(TypedDict, total=False): + cookies: List[Cookie] + origins: List[OriginState] + + +class DomainState(BaseModel): + domain: str + storage_state: StorageState + + +class AuthSession(BaseModel): + user_agent: Optional[str] + domain_states: List[DomainState] + + def to_storage_state(self) -> StorageState: + cookies = [] + origins = [] + for domain_state in self.domain_states: + cookies.extend(domain_state.storage_state.get("cookies", [])) + origins.extend(domain_state.storage_state.get("origins", [])) + return StorageState(cookies=cookies, origins=origins) diff --git a/dendrite/browser/sync_api/_core/models/download_interface.py b/dendrite/browser/sync_api/_core/models/download_interface.py new file mode 100644 index 0000000..8a68843 --- /dev/null +++ b/dendrite/browser/sync_api/_core/models/download_interface.py @@ -0,0 +1,20 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, Union +from playwright.sync_api import Download + + +class DownloadInterface(ABC, Download): + + def __init__(self, download: Download): + self._download = download + + def __getattribute__(self, name: str) -> Any: + try: + return super().__getattribute__(name) + except AttributeError: + return getattr(self._download, name) + + @abstractmethod + def save_as(self, path: Union[str, Path]) -> None: + pass diff --git a/dendrite/browser/sync_api/_core/models/page_diff_information.py b/dendrite/browser/sync_api/_core/models/page_diff_information.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/_core/models/page_information.py b/dendrite/browser/sync_api/_core/models/page_information.py new file mode 100644 index 0000000..788dc87 --- /dev/null +++ b/dendrite/browser/sync_api/_core/models/page_information.py @@ -0,0 +1,15 @@ +from typing import Optional +from pydantic import BaseModel +from typing_extensions import TypedDict + + +class InteractableElementInfo(TypedDict): + attrs: Optional[str] + text: Optional[str] + + +class PageInformation(BaseModel): + url: str + raw_html: str + screenshot_base64: str + time_since_frame_navigated: float diff --git a/dendrite/browser/sync_api/_core/models/response.py b/dendrite/browser/sync_api/_core/models/response.py new file mode 100644 index 0000000..50d9a23 --- /dev/null +++ b/dendrite/browser/sync_api/_core/models/response.py @@ -0,0 +1,54 @@ +from typing import Dict, Iterator +from dendrite.browser.sync_api._core.dendrite_element import Element + + +class ElementsResponse: + """ + ElementsResponse is a class that encapsulates a dictionary of Dendrite elements, + allowing for attribute-style access and other convenient interactions. + + This class is used to store and access the elements retrieved by the `get_elements` function. + The attributes of this class dynamically match the keys of the dictionary passed to the `get_elements` function, + allowing for direct attribute-style access to the corresponding `Element` objects. + + Attributes: + _data (Dict[str, Element]): A dictionary where keys are the names of elements and values are the corresponding `Element` objects. + + Args: + data (Dict[str, Element]): The dictionary of elements to be encapsulated by the class. + + Methods: + __getattr__(name: str) -> Element: + Allows attribute-style access to the elements in the dictionary. + + __getitem__(key: str) -> Element: + Enables dictionary-style access to the elements. + + __iter__() -> Iterator[str]: + Provides an iterator over the keys in the dictionary. + + __repr__() -> str: + Returns a string representation of the class instance. + """ + + _data: Dict[str, Element] + + def __init__(self, data: Dict[str, Element]): + self._data = data + + def __getattr__(self, name: str) -> Element: + try: + return self._data[name] + except KeyError: + raise AttributeError( + f"'{self.__class__.__name__}' object has no attribute '{name}'" + ) + + def __getitem__(self, key: str) -> Element: + return self._data[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._data) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self._data})" diff --git a/dendrite/browser/sync_api/_core/protocol/page_protocol.py b/dendrite/browser/sync_api/_core/protocol/page_protocol.py new file mode 100644 index 0000000..f74afff --- /dev/null +++ b/dendrite/browser/sync_api/_core/protocol/page_protocol.py @@ -0,0 +1,20 @@ +from typing import TYPE_CHECKING, Protocol +from dendrite.logic.hosted._api.browser_api_client import BrowserAPIClient +from dendrite.logic.interfaces import SyncProtocol + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite + from dendrite.browser.sync_api._core.dendrite_page import Page + + +class DendritePageProtocol(Protocol): + """ + Protocol that specifies the required methods and attributes + for the `ExtractionMixin` to work. + """ + + def _get_dendrite_browser(self) -> "Dendrite": ... + + def _get_logic_api(self) -> SyncProtocol: ... + + def _get_page(self) -> "Page": ... diff --git a/dendrite/browser/sync_api/_dom/__init__.py b/dendrite/browser/sync_api/_dom/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/_remote_impl/__init__.py b/dendrite/browser/sync_api/_remote_impl/__init__.py new file mode 100644 index 0000000..4d00d3c --- /dev/null +++ b/dendrite/browser/sync_api/_remote_impl/__init__.py @@ -0,0 +1,3 @@ +from .browserbase import BrowserbaseDownload + +__all__ = ["BrowserbaseDownload"] diff --git a/dendrite/browser/sync_api/_remote_impl/browserbase/__init__.py b/dendrite/browser/sync_api/_remote_impl/browserbase/__init__.py new file mode 100644 index 0000000..eb977c7 --- /dev/null +++ b/dendrite/browser/sync_api/_remote_impl/browserbase/__init__.py @@ -0,0 +1,3 @@ +from ._download import BrowserbaseDownload + +__all__ = ["BrowserbaseDownload"] diff --git a/dendrite/browser/sync_api/_remote_impl/browserbase/_client.py b/dendrite/browser/sync_api/_remote_impl/browserbase/_client.py new file mode 100644 index 0000000..ddc6831 --- /dev/null +++ b/dendrite/browser/sync_api/_remote_impl/browserbase/_client.py @@ -0,0 +1,63 @@ +import time +import time +from pathlib import Path +from typing import Optional, Union +import httpx +from loguru import logger +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException + + +class BrowserbaseClient: + + def __init__(self, api_key: str, project_id: str) -> None: + self.api_key = api_key + self.project_id = project_id + + def create_session(self) -> str: + logger.debug("Creating session") + "\n Creates a session using the Browserbase API.\n\n Returns:\n str: The ID of the created session.\n " + url = "https://www.browserbase.com/v1/sessions" + headers = {"Content-Type": "application/json", "x-bb-api-key": self.api_key} + json = {"projectId": self.project_id, "keepAlive": False} + response = httpx.post(url, json=json, headers=headers) + if response.status_code >= 400: + raise DendriteException(f"Failed to create session: {response.text}") + return response.json()["id"] + + def stop_session(self, session_id: str): + url = f"https://www.browserbase.com/v1/sessions/{session_id}" + headers = {"Content-Type": "application/json", "x-bb-api-key": self.api_key} + json = {"projectId": self.project_id, "status": "REQUEST_RELEASE"} + with httpx.Client() as client: + response = client.post(url, json=json, headers=headers) + return response.json() + + def connect_url(self, enable_proxy: bool, session_id: Optional[str] = None) -> str: + url = f"wss://connect.browserbase.com?apiKey={self.api_key}" + if session_id: + url += f"&sessionId={session_id}" + if enable_proxy: + url += "&enableProxy=true" + return url + + def save_downloads_on_disk( + self, session_id: str, path: Union[str, Path], retry_for_seconds: float + ): + url = f"https://www.browserbase.com/v1/sessions/{session_id}/downloads" + headers = {"x-bb-api-key": self.api_key} + file_path = Path(path) + with httpx.Client() as session: + timeout = time.time() + retry_for_seconds + while time.time() < timeout: + try: + response = session.get(url, headers=headers) + if response.status_code == 200: + array_buffer = response.read() + if len(array_buffer) > 0: + with open(file_path, "wb") as f: + f.write(array_buffer) + return + except Exception as e: + logger.debug(f"Error fetching downloads: {e}") + time.sleep(2) + logger.debug("Failed to download files within the time limit.") diff --git a/dendrite/browser/sync_api/_remote_impl/browserbase/_download.py b/dendrite/browser/sync_api/_remote_impl/browserbase/_download.py new file mode 100644 index 0000000..24647c4 --- /dev/null +++ b/dendrite/browser/sync_api/_remote_impl/browserbase/_download.py @@ -0,0 +1,53 @@ +import re +import shutil +import zipfile +from pathlib import Path +from typing import Union +from loguru import logger +from playwright.sync_api import Download +from dendrite.browser.sync_api._core.models.download_interface import DownloadInterface +from dendrite.browser.sync_api._remote_impl.browserbase._client import BrowserbaseClient + + +class BrowserbaseDownload(DownloadInterface): + + def __init__( + self, session_id: str, download: Download, client: BrowserbaseClient + ) -> None: + super().__init__(download) + self._session_id = session_id + self._client = client + + def save_as(self, path: Union[str, Path], timeout: float = 20) -> None: + """ + Save the latest file from the downloaded ZIP archive to the specified path. + + Args: + path (Union[str, Path]): The destination file path where the latest file will be saved. + timeout (float, optional): Timeout for the save operation. Defaults to 20 seconds. + + Raises: + Exception: If no matching files are found in the ZIP archive or if the file cannot be saved. + """ + destination_path = Path(path) + source_path = self._download.path() + destination_path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(source_path, "r") as zip_ref: + file_list = zip_ref.namelist() + sorted_files = sorted(file_list, key=extract_timestamp, reverse=True) + if not sorted_files: + raise FileNotFoundError( + "No files found in the Browserbase download ZIP" + ) + latest_file = sorted_files[0] + with zip_ref.open(latest_file) as source, open( + destination_path, "wb" + ) as target: + shutil.copyfileobj(source, target) + logger.info(f"Latest file saved successfully to {destination_path}") + + +def extract_timestamp(filename): + timestamp_pattern = re.compile("-(\\d+)\\.") + match = timestamp_pattern.search(filename) + return int(match.group(1)) if match else 0 diff --git a/dendrite/browser/sync_api/_remote_impl/browserbase/_impl.py b/dendrite/browser/sync_api/_remote_impl/browserbase/_impl.py new file mode 100644 index 0000000..3a94417 --- /dev/null +++ b/dendrite/browser/sync_api/_remote_impl/browserbase/_impl.py @@ -0,0 +1,64 @@ +from typing import TYPE_CHECKING, Optional +from dendrite.browser._common._exceptions.dendrite_exception import ( + BrowserNotLaunchedError, +) +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +from loguru import logger +from playwright.sync_api import Playwright +from dendrite.browser.sync_api._remote_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.sync_api._remote_impl.browserbase._download import ( + BrowserbaseDownload, +) + + +class BrowserBaseImpl(ImplBrowser): + + def __init__(self, settings: BrowserbaseConfig) -> None: + self.settings = settings + self._client = BrowserbaseClient( + self.settings.api_key, self.settings.project_id + ) + self._session_id: Optional[str] = None + + def stop_session(self): + if self._session_id: + self._client.stop_session(self._session_id) + + def start_browser(self, playwright: Playwright, pw_options: dict): + logger.debug("Starting browser") + self._session_id = self._client.create_session() + url = self._client.connect_url(self.settings.enable_proxy, self._session_id) + logger.debug(f"Connecting to browser at {url}") + return playwright.chromium.connect_over_cdp(url) + + def configure_context(self, browser: "Dendrite"): + logger.debug("Configuring browser context") + page = browser.get_active_page() + pw_page = page.playwright_page + if browser.browser_context is None: + raise BrowserNotLaunchedError() + client = browser.browser_context.new_cdp_session(pw_page) + client.send( + "Browser.setDownloadBehavior", + {"behavior": "allow", "downloadPath": "downloads", "eventsEnabled": True}, + ) + + def get_download( + self, + dendrite_browser: "Dendrite", + pw_page: PlaywrightPage, + timeout: float = 30000, + ) -> BrowserbaseDownload: + if not self._session_id: + raise ValueError( + "Downloads are not enabled for this provider. Specify enable_downloads=True in the constructor" + ) + logger.debug("Getting download") + download = dendrite_browser._download_handler.get_data(pw_page, timeout) + self._client.save_downloads_on_disk(self._session_id, download.path(), 30) + return BrowserbaseDownload(self._session_id, download, self._client) diff --git a/dendrite/browser/sync_api/_remote_impl/browserless/__init__.py b/dendrite/browser/sync_api/_remote_impl/browserless/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/_remote_impl/browserless/_impl.py b/dendrite/browser/sync_api/_remote_impl/browserless/_impl.py new file mode 100644 index 0000000..d254015 --- /dev/null +++ b/dendrite/browser/sync_api/_remote_impl/browserless/_impl.py @@ -0,0 +1,57 @@ +import json +from typing import TYPE_CHECKING, Optional +from dendrite.browser._common._exceptions.dendrite_exception import ( + BrowserNotLaunchedError, +) +from dendrite.browser.sync_api._core._impl_browser import ImplBrowser +from dendrite.browser.sync_api._core._type_spec import PlaywrightPage +from dendrite.browser.remote.browserless_config import BrowserlessConfig + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_browser import Dendrite +import urllib.parse +from loguru import logger +from playwright.sync_api import Playwright +from dendrite.browser.sync_api._remote_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.sync_api._remote_impl.browserbase._download import ( + BrowserbaseDownload, +) + + +class BrowserlessImpl(ImplBrowser): + + def __init__(self, settings: BrowserlessConfig) -> None: + self.settings = settings + self._session_id: Optional[str] = None + + def stop_session(self): + pass + + def start_browser(self, playwright: Playwright, pw_options: dict): + logger.debug("Starting browser") + url = self._format_connection_url(pw_options) + logger.debug(f"Connecting to browser at {url}") + return playwright.chromium.connect_over_cdp(url) + + def _format_connection_url(self, pw_options: dict) -> str: + url = self.settings.url.rstrip("?").rstrip("/") + query = { + "token": self.settings.api_key, + "blockAds": self.settings.block_ads, + "launch": json.dumps(pw_options), + } + if self.settings.proxy: + query["proxy"] = (self.settings.proxy,) + query["proxyCountry"] = (self.settings.proxy_country,) + return f"{url}?{urllib.parse.urlencode(query)}" + + def configure_context(self, browser: "Dendrite"): + pass + + def get_download( + self, + dendrite_browser: "Dendrite", + pw_page: PlaywrightPage, + timeout: float = 30000, + ) -> BrowserbaseDownload: + raise NotImplementedError("Downloads are not supported for Browserless") diff --git a/dendrite/logic/factory.py b/dendrite/logic/factory.py index 02f2782..d4fd85f 100644 --- a/dendrite/logic/factory.py +++ b/dendrite/logic/factory.py @@ -1,6 +1,6 @@ # from typing import Literal, Optional -# from dendrite.logic.interfaces.async_api import LogicAPIProtocol +# from dendrite.logic.interfaces import AsyncProtocol # class BrowserAPIFactory: diff --git a/dendrite/logic/interfaces/__init__.py b/dendrite/logic/interfaces/__init__.py new file mode 100644 index 0000000..3f4d492 --- /dev/null +++ b/dendrite/logic/interfaces/__init__.py @@ -0,0 +1,8 @@ +from .async_api import AsyncProtocol +from .sync_api import SyncProtocol + + +__all__ = [ + "AsyncProtocol", + "SyncProtocol", +] diff --git a/dendrite/logic/interfaces/async_api.py b/dendrite/logic/interfaces/async_api.py index fb36946..646b2eb 100644 --- a/dendrite/logic/interfaces/async_api.py +++ b/dendrite/logic/interfaces/async_api.py @@ -28,7 +28,7 @@ async def extract(self, dto: ExtractDTO) -> ExtractResponse: ... async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... -class LocalProtocol(LogicAPIProtocol): +class AsyncProtocol(LogicAPIProtocol): async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: return await get_element.get_element(dto) diff --git a/dendrite/logic/interfaces/sync_api.py b/dendrite/logic/interfaces/sync_api.py index 8b2f8f6..4b75fc2 100644 --- a/dendrite/logic/interfaces/sync_api.py +++ b/dendrite/logic/interfaces/sync_api.py @@ -59,7 +59,7 @@ def extract(self, dto: ExtractDTO) -> ExtractResponse: ... def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... -class LocalProtocol(LogicAPIProtocol): +class SyncProtocol(LogicAPIProtocol): def get_element(self, dto: GetElementsDTO) -> GetElementResponse: return run_coroutine_sync(get_element.get_element(dto)) diff --git a/dendrite/logic/llm/config.py b/dendrite/logic/llm/config.py index 49def1b..4f70e16 100644 --- a/dendrite/logic/llm/config.py +++ b/dendrite/logic/llm/config.py @@ -7,8 +7,16 @@ except ModuleNotFoundError: import tomli as tomllib # type: ignore # tomllib is only included standard lib for python 3.11+ - -DEFAULT_LLM = { +AGENTS = Literal[ + "extract_agent", + "scroll_agent", + "ask_page_agent", + "segment_agent", + "select_agent", + "verify_action_agent", +] + +DEFAULT_LLM: Dict[str, LLM] = { "extract_agent": LLM( "claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500 ), diff --git a/poetry.lock b/poetry.lock index 1fc589e..e5d6827 100644 --- a/poetry.lock +++ b/poetry.lock @@ -214,6 +214,21 @@ docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphi tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +[[package]] +name = "autoflake" +version = "2.3.1" +description = "Removes unused imports and unused variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "autoflake-2.3.1-py3-none-any.whl", hash = "sha256:3ae7495db9084b7b32818b4140e6dc4fc280b712fb414f5b8fe57b0a8e85a840"}, + {file = "autoflake-2.3.1.tar.gz", hash = "sha256:c98b75dc5b0a86459c4f01a1d32ac7eb4338ec4317a4469515ff1e687ecd909e"}, +] + +[package.dependencies] +pyflakes = ">=3.0.0" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + [[package]] name = "autopep8" version = "2.3.1" @@ -1058,13 +1073,13 @@ referencing = ">=0.31.0" [[package]] name = "litellm" -version = "1.52.6" +version = "1.52.9" description = "Library to easily interface with LLM API providers" optional = false python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" files = [ - {file = "litellm-1.52.6-py3-none-any.whl", hash = "sha256:9b3e9fb51f7e2a3cc8b50997b346c55aae9435a138d9a656f18e262750a1bfe1"}, - {file = "litellm-1.52.6.tar.gz", hash = "sha256:d67c653f97bd07f503b975c167de1e25632b7bc6bb3c008c46921e4acc81ec60"}, + {file = "litellm-1.52.9-py3-none-any.whl", hash = "sha256:a1ef5561d220d77059a359da497f0ab04c721205c6795f151b07be5bbe51fe45"}, + {file = "litellm-1.52.9.tar.gz", hash = "sha256:73a05fed76cfac4357ee4117f28608209db891223fb9c6e03dddfe1723666437"}, ] [package.dependencies] @@ -2781,4 +2796,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "80acd51d9be644348cf82ebcd258b260ea43a8a30b5d3025af70f75e5961c5b6" +content-hash = "f08f549a61b2efdf0e4a859882fe6009ecc47d4cddff3fbdffec73e7ed5ad28a" diff --git a/pyproject.toml b/pyproject.toml index faaf374..897d9f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ typing-extensions = "^4.12.0" loguru = "^0.7.2" httpx = "^0.27.2" markdownify = "^0.13.1" -litellm = "^1.52.6" +litellm = "^1.52.9" pillow = "^11.0.0" json-repair = "^0.30.1" @@ -48,13 +48,15 @@ pylint-pydantic = "^0.3.2" flake8 = "^7.1.1" pytest = "^8.3.3" pytest-asyncio = "^0.24.0" +autoflake = "^2.3.1" +isort = "^5.13.2" [tool.pylint] load-plugins = "pylint_pydantic" [tool.poe.tasks] generate_sync = "python scripts/generate_sync.py" -format_sync = "python -m black ./dendrite/sync_api/" +format_sync = "python -m black ./dendrite//browser/sync_api/" build_sync = ["generate_sync", "format_sync"] test_sync = "pytest tests/tests_sync" diff --git a/scripts/generate_sync.py b/scripts/generate_sync.py index 8bdee38..7d607a9 100644 --- a/scripts/generate_sync.py +++ b/scripts/generate_sync.py @@ -279,8 +279,8 @@ def get_uncommitted_diff(folder): if __name__ == "__main__": - source_dir = "dendrite/async_api" - target_dir = "dendrite/sync_api" + source_dir = "dendrite/browser/async_api" + target_dir = "dendrite/browser/sync_api" renames = { "AsyncBrowserbaseDownload": "BrowserbaseDownload", "AsyncBrowserbaseBrowser": "BrowserbaseBrowser", @@ -289,6 +289,7 @@ def get_uncommitted_diff(folder): "AsyncPage": "Page", "AsyncDendriteRemoteBrowser": "DendriteRemoteBrowser", "AsyncElementsResponse": "ElementsResponse", + "AsyncProtocol": "SyncProtocol", } if check_for_uncommitted_changes(target_dir): From 19147f76139675b8183b9311fc89038c7d4808b5 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Tue, 3 Dec 2024 10:52:39 +0100 Subject: [PATCH 07/18] Refactor element retrieval logic to utilize a unified Config class for managing configurations across async and sync APIs. Updated get_element and related functions to support cached selectors and improved error handling. Removed deprecated code and streamlined selector handling in both async and sync contexts. Added new CachedSelectorDTO for better cache management. --- dendrite/__init__.py | 3 + dendrite/browser/async_api/_core/_utils.py | 30 +-- .../async_api/_core/dendrite_browser.py | 6 +- .../async_api/_core/mixin/get_element.py | 216 ++++++++++++------ dendrite/browser/sync_api/_core/_utils.py | 29 +-- .../sync_api/_core/dendrite_browser.py | 6 +- .../sync_api/_core/mixin/get_element.py | 186 +++++++++------ dendrite/logic/ask/ask.py | 8 +- dendrite/logic/config.py | 15 +- dendrite/logic/extract/extract.py | 15 +- dendrite/logic/extract/extract_agent.py | 20 +- dendrite/logic/extract/scroll_agent.py | 4 +- .../logic/get_element/agents/segment_agent.py | 8 +- .../logic/get_element/agents/select_agent.py | 3 +- dendrite/logic/get_element/cached_selector.py | 13 +- dendrite/logic/get_element/get_element.py | 79 +++---- dendrite/logic/get_element/hanifi_search.py | 14 +- dendrite/logic/interfaces/async_api.py | 24 +- dendrite/logic/interfaces/sync_api.py | 23 +- dendrite/logic/llm/config.py | 9 - dendrite/logic/verify_interaction.py | 6 +- dendrite/models/dto/cached_selector_dto.py | 6 + dendrite/models/dto/get_elements_dto.py | 2 - .../models/response/get_element_response.py | 3 +- 24 files changed, 413 insertions(+), 315 deletions(-) create mode 100644 dendrite/models/dto/cached_selector_dto.py diff --git a/dendrite/__init__.py b/dendrite/__init__.py index 979e742..b0777ff 100644 --- a/dendrite/__init__.py +++ b/dendrite/__init__.py @@ -14,6 +14,8 @@ ElementsResponse, ) +from dendrite.logic.config import Config + # logger.remove() # fmt = "{time: HH:mm:ss.SSS} | {level: <8}- {message}" @@ -30,4 +32,5 @@ "Element", "Page", "ElementsResponse", + "Config", ] diff --git a/dendrite/browser/async_api/_core/_utils.py b/dendrite/browser/async_api/_core/_utils.py index 62fd9f6..1db3237 100644 --- a/dendrite/browser/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_core/_utils.py @@ -8,6 +8,7 @@ from dendrite.browser.async_api._core.dendrite_element import AsyncElement from dendrite.browser.async_api._core.models.response import AsyncElementsResponse from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.selector import Selector if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_page import AsyncPage @@ -102,27 +103,16 @@ async def _get_all_elements_from_selector_soup( async def get_elements_from_selectors_soup( page: "AsyncPage", soup: BeautifulSoup, - res: GetElementResponse, + selectors: List[Selector], only_one: bool, -) -> Union[Optional[AsyncElement], List[AsyncElement], AsyncElementsResponse]: - if isinstance(res.selectors, dict): - result = {} - for key, selectors in res.selectors.items(): - for selector in selectors: - dendrite_elements = await _get_all_elements_from_selector_soup( - selector, soup, page - ) - if len(dendrite_elements) > 0: - result[key] = dendrite_elements[0] - break - return AsyncElementsResponse(result) - elif isinstance(res.selectors, list): - for selector in reversed(res.selectors): - dendrite_elements = await _get_all_elements_from_selector_soup( - selector, soup, page - ) +) -> Union[Optional[AsyncElement], List[AsyncElement]]: + + for selector in reversed(selectors): + dendrite_elements = await _get_all_elements_from_selector_soup( + selector.selector, soup, page + ) - if len(dendrite_elements) > 0: - return dendrite_elements[0] if only_one else dendrite_elements + if len(dendrite_elements) > 0: + return dendrite_elements[0] if only_one else dendrite_elements return None diff --git a/dendrite/browser/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/_core/dendrite_browser.py index 6bd4b00..8f902cc 100644 --- a/dendrite/browser/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/_core/dendrite_browser.py @@ -22,7 +22,6 @@ IncorrectOutcomeError, ) from dendrite.browser._common.constants import STEALTH_ARGS -from dendrite.models.api_config import APIConfig from dendrite.browser.async_api._core._impl_browser import ImplBrowser from dendrite.browser.async_api._core._impl_mapping import get_impl from dendrite.browser.async_api._core._managers.page_manager import PageManager @@ -41,6 +40,7 @@ WaitForMixin, ) from dendrite.browser.remote import Providers +from dendrite.logic.config import Config from dendrite.logic.interfaces import AsyncProtocol @@ -86,6 +86,7 @@ def __init__( "args": STEALTH_ARGS, }, remote_config: Optional[Providers] = None, + config: Optional[Config] = None, ): """ Initializes AsyncDendrite with API keys and Playwright options. @@ -119,7 +120,8 @@ def __init__( self._upload_handler = EventSync(event_type=FileChooser) self._download_handler = EventSync(event_type=Download) self.closed = False - self._browser_api_client: AsyncProtocol = AsyncProtocol() + self._config = config or Config() + self._browser_api_client: AsyncProtocol = AsyncProtocol(self._config) @property def pages(self) -> List[AsyncPage]: diff --git a/dendrite/browser/async_api/_core/mixin/get_element.py b/dendrite/browser/async_api/_core/mixin/get_element.py index bb171db..5e9245a 100644 --- a/dendrite/browser/async_api/_core/mixin/get_element.py +++ b/dendrite/browser/async_api/_core/mixin/get_element.py @@ -1,15 +1,30 @@ import asyncio import time -from typing import Dict, List, Literal, Optional, Union, overload - +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Literal, + Optional, + Union, + overload, +) + +from bs4 import BeautifulSoup from loguru import logger -from dendrite.browser.async_api._core._utils import get_elements_from_selectors_soup +from dendrite.browser.async_api._core._utils import _get_all_elements_from_selector_soup from dendrite.browser.async_api._core.dendrite_element import AsyncElement + +if TYPE_CHECKING: + from dendrite.browser.async_api._core.dendrite_page import AsyncPage from dendrite.browser.async_api._core.models.response import AsyncElementsResponse from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.models.dto.get_elements_dto import CheckSelectorCacheDTO, GetElementsDTO -from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO +from dendrite.models.dto.get_elements_dto import GetElementsDTO + CACHE_TIMEOUT = 5 @@ -196,38 +211,29 @@ async def _get_element( Union[AsyncElement, List[AsyncElement], AsyncElementsResponse]: The retrieved element, list of elements, or response object. """ - start_time = time.time() + if isinstance(prompt_or_elements, Dict): + return None - # First, let's check if there is a cached selector + start_time = time.time() page = await self._get_page() + soup = await page._get_soup() - # If we have cached elements, attempt to use them with an exponentation backoff - if use_cache == True: - logger.debug("Attempting to use cached selectors") - res = await attempt_with_backoff( - self, - prompt_or_elements, - only_one, - remaining_timeout=CACHE_TIMEOUT, - only_use_cache=True, + if use_cache: + cached_elements = await self._try_cached_selectors( + page, soup, prompt_or_elements, only_one ) - if res: - return res - else: - logger.debug( - f"After attempting to use cached selectors several times without success, let's find the elements using the find element agent." - ) + if cached_elements: + return cached_elements - # Now that no cached selectors were found or they failed repeatedly, let's use the find element agent to find the requested elements. + # Now that no cached selectors were found or they failed repeatedly, let's use the find element agent logger.info( "Proceeding to use the find element agent to find the requested elements." ) - res = await attempt_with_backoff( + res = await try_get_element( self, prompt_or_elements, only_one, remaining_timeout=timeout - (time.time() - start_time), - only_use_cache=False, ) if res: return res @@ -237,37 +243,109 @@ async def _get_element( ) return None + async def _try_cached_selectors( + self, + page: "AsyncPage", + soup: BeautifulSoup, + prompt: str, + only_one: bool, + ) -> Union[Optional[AsyncElement], List[AsyncElement]]: + """ + Attempts to retrieve elements using cached selectors with exponential backoff. -async def attempt_with_backoff( - obj: DendritePageProtocol, - prompt_or_elements: Union[str, Dict[str, str]], - only_one: bool, - remaining_timeout: float, - only_use_cache: bool = False, -) -> Union[Optional[AsyncElement], List[AsyncElement], AsyncElementsResponse]: - TIMEOUT_INTERVAL: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0] + Args: + page: The current page object + soup: The BeautifulSoup object of the current page + prompt: The prompt to search for + only_one: Whether to return only one element + + Returns: + The found elements if successful, None otherwise + """ + dto = CachedSelectorDTO(url=page.url, prompt=prompt) + selectors = await self._get_logic_api().get_cached_selectors(dto) + + if len(selectors) == 0: + logger.debug("No cached selectors found") + return None + + logger.debug("Attempting to use cached selectors with backoff") + str_selectors = list(map(lambda x: x.selector, selectors)) + + async def try_cached_selectors(): + return await get_elements_from_selectors_soup( + page, soup, str_selectors, only_one + ) + + return await _attempt_with_backoff_helper( + "cached_selectors", + try_cached_selectors, + timeout=CACHE_TIMEOUT, + ) + + +async def _attempt_with_backoff_helper( + operation_name: str, + operation: Callable, + timeout: float, + backoff_intervals: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0], +) -> Optional[Any]: + """ + Generic helper function that implements exponential backoff for operations. + + Args: + operation_name: Name of the operation for logging + operation: Async function to execute + timeout: Maximum time to spend attempting the operation + backoff_intervals: List of timeouts between attempts + + Returns: + The result of the operation if successful, None otherwise + """ total_elapsed_time = 0 start_time = time.time() - for current_timeout in TIMEOUT_INTERVAL: - if total_elapsed_time >= remaining_timeout: + for i, current_timeout in enumerate(backoff_intervals): + if total_elapsed_time >= timeout: logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") return None request_start_time = time.time() - page = await obj._get_page() - page_information = await page.get_page_information( - include_screenshot=not only_use_cache + result = await operation() + request_duration = time.time() - request_start_time + + if result: + return result + + sleep_duration = max(0, current_timeout - request_duration) + logger.info( + f"{operation_name} attempt {i+1} failed. Sleeping for {sleep_duration:.2f} seconds" ) + await asyncio.sleep(sleep_duration) + total_elapsed_time = time.time() - start_time + + logger.error( + f"All {operation_name} attempts failed after {total_elapsed_time:.2f} seconds" + ) + return None + + +async def try_get_element( + obj: DendritePageProtocol, + prompt_or_elements: Union[str, Dict[str, str]], + only_one: bool, + remaining_timeout: float, +) -> Union[Optional[AsyncElement], List[AsyncElement], AsyncElementsResponse]: + + async def _try_get_element(): + page = await obj._get_page() + page_information = await page.get_page_information() dto = GetElementsDTO( page_information=page_information, prompt=prompt_or_elements, - use_cache=only_use_cache, only_one=only_one, - force_use_cache=only_use_cache, ) res = await obj._get_logic_api().get_element(dto) - request_duration = time.time() - request_start_time if res.status == "impossible": logger.error( @@ -277,42 +355,32 @@ async def attempt_with_backoff( if res.status == "success": logger.success(f"d[id]: {res.d_id} Selectors:{res.selectors}") - response = await get_elements_from_selectors_soup( - page, await page._get_previous_soup(), res, only_one - ) - if response: - return response + if res.selectors is not None: + return await get_elements_from_selectors_soup( + page, await page._get_previous_soup(), res.selectors, only_one + ) + return None - sleep_duration = max(0, current_timeout - request_duration) - logger.info( - f"Failed to get elements for prompt:\n\n'{prompt_or_elements}'\n\nStatus: {res.status}\n\nMessage: {res.message}\n\nSleeping for {sleep_duration:.2f} seconds" - ) - await asyncio.sleep(sleep_duration) - total_elapsed_time = time.time() - start_time + return await _attempt_with_backoff_helper( + "find_element_agent", + _try_get_element, + remaining_timeout, + ) - logger.error(f"All attempts failed after {total_elapsed_time:.2f} seconds") - return None +async def get_elements_from_selectors_soup( + page: "AsyncPage", + soup: BeautifulSoup, + selectors: List[str], + only_one: bool, +) -> Union[Optional[AsyncElement], List[AsyncElement]]: -async def get_elements_from_selectors( - obj: DendritePageProtocol, res: GetElementResponse, only_one: bool -) -> Union[Optional[AsyncElement], List[AsyncElement], AsyncElementsResponse]: - if isinstance(res.selectors, dict): - result = {} - for key, selectors in res.selectors.items(): - for selector in selectors: - page = await obj._get_page() - dendrite_elements = await page._get_all_elements_from_selector(selector) - if len(dendrite_elements) > 0: - result[key] = dendrite_elements[0] - break - return AsyncElementsResponse(result) - elif isinstance(res.selectors, list): - for selector in reversed(res.selectors): - page = await obj._get_page() - dendrite_elements = await page._get_all_elements_from_selector(selector) - - if len(dendrite_elements) > 0: - return dendrite_elements[0] if only_one else dendrite_elements + for selector in reversed(selectors): + dendrite_elements = await _get_all_elements_from_selector_soup( + selector, soup, page + ) + + if len(dendrite_elements) > 0: + return dendrite_elements[0] if only_one else dendrite_elements return None diff --git a/dendrite/browser/sync_api/_core/_utils.py b/dendrite/browser/sync_api/_core/_utils.py index 37f328e..846ee40 100644 --- a/dendrite/browser/sync_api/_core/_utils.py +++ b/dendrite/browser/sync_api/_core/_utils.py @@ -6,6 +6,7 @@ from dendrite.browser.sync_api._core.dendrite_element import Element from dendrite.browser.sync_api._core.models.response import ElementsResponse from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.selector import Selector if TYPE_CHECKING: from dendrite.browser.sync_api._core.dendrite_page import Page @@ -81,24 +82,12 @@ def _get_all_elements_from_selector_soup( def get_elements_from_selectors_soup( - page: "Page", soup: BeautifulSoup, res: GetElementResponse, only_one: bool -) -> Union[Optional[Element], List[Element], ElementsResponse]: - if isinstance(res.selectors, dict): - result = {} - for key, selectors in res.selectors.items(): - for selector in selectors: - dendrite_elements = _get_all_elements_from_selector_soup( - selector, soup, page - ) - if len(dendrite_elements) > 0: - result[key] = dendrite_elements[0] - break - return ElementsResponse(result) - elif isinstance(res.selectors, list): - for selector in reversed(res.selectors): - dendrite_elements = _get_all_elements_from_selector_soup( - selector, soup, page - ) - if len(dendrite_elements) > 0: - return dendrite_elements[0] if only_one else dendrite_elements + page: "Page", soup: BeautifulSoup, selectors: List[Selector], only_one: bool +) -> Union[Optional[Element], List[Element]]: + for selector in reversed(selectors): + dendrite_elements = _get_all_elements_from_selector_soup( + selector.selector, soup, page + ) + if len(dendrite_elements) > 0: + return dendrite_elements[0] if only_one else dendrite_elements return None diff --git a/dendrite/browser/sync_api/_core/dendrite_browser.py b/dendrite/browser/sync_api/_core/dendrite_browser.py index 4bf8316..b37e35c 100644 --- a/dendrite/browser/sync_api/_core/dendrite_browser.py +++ b/dendrite/browser/sync_api/_core/dendrite_browser.py @@ -20,7 +20,6 @@ IncorrectOutcomeError, ) from dendrite.browser._common.constants import STEALTH_ARGS -from dendrite.models.api_config import APIConfig from dendrite.browser.sync_api._core._impl_browser import ImplBrowser from dendrite.browser.sync_api._core._impl_mapping import get_impl from dendrite.browser.sync_api._core._managers.page_manager import PageManager @@ -39,6 +38,7 @@ WaitForMixin, ) from dendrite.browser.remote import Providers +from dendrite.logic.config import Config from dendrite.logic.interfaces import SyncProtocol @@ -81,6 +81,7 @@ def __init__( self, playwright_options: Any = {"headless": False, "args": STEALTH_ARGS}, remote_config: Optional[Providers] = None, + config: Optional[Config] = None, ): """ Initializes Dendrite with API keys and Playwright options. @@ -104,7 +105,8 @@ def __init__( self._upload_handler = EventSync(event_type=FileChooser) self._download_handler = EventSync(event_type=Download) self.closed = False - self._browser_api_client: SyncProtocol = SyncProtocol() + self._config = config or Config() + self._browser_api_client: SyncProtocol = SyncProtocol(self._config) @property def pages(self) -> List[Page]: diff --git a/dendrite/browser/sync_api/_core/mixin/get_element.py b/dendrite/browser/sync_api/_core/mixin/get_element.py index 1d4694b..22b7583 100644 --- a/dendrite/browser/sync_api/_core/mixin/get_element.py +++ b/dendrite/browser/sync_api/_core/mixin/get_element.py @@ -1,13 +1,27 @@ import time import time -from typing import Dict, List, Literal, Optional, Union, overload +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Literal, + Optional, + Union, + overload, +) +from bs4 import BeautifulSoup from loguru import logger -from dendrite.browser.sync_api._core._utils import get_elements_from_selectors_soup +from dendrite.browser.sync_api._core._utils import _get_all_elements_from_selector_soup from dendrite.browser.sync_api._core.dendrite_element import Element + +if TYPE_CHECKING: + from dendrite.browser.sync_api._core.dendrite_page import Page from dendrite.browser.sync_api._core.models.response import ElementsResponse from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.models.dto.get_elements_dto import CheckSelectorCacheDTO, GetElementsDTO -from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO +from dendrite.models.dto.get_elements_dto import GetElementsDTO CACHE_TIMEOUT = 5 @@ -179,32 +193,25 @@ def _get_element( Returns: Union[Element, List[Element], ElementsResponse]: The retrieved element, list of elements, or response object. """ + if isinstance(prompt_or_elements, Dict): + return None start_time = time.time() page = self._get_page() - if use_cache == True: - logger.debug("Attempting to use cached selectors") - res = attempt_with_backoff( - self, - prompt_or_elements, - only_one, - remaining_timeout=CACHE_TIMEOUT, - only_use_cache=True, + soup = page._get_soup() + if use_cache: + cached_elements = self._try_cached_selectors( + page, soup, prompt_or_elements, only_one ) - if res: - return res - else: - logger.debug( - f"After attempting to use cached selectors several times without success, let's find the elements using the find element agent." - ) + if cached_elements: + return cached_elements logger.info( "Proceeding to use the find element agent to find the requested elements." ) - res = attempt_with_backoff( + res = try_get_element( self, prompt_or_elements, only_one, remaining_timeout=timeout - (time.time() - start_time), - only_use_cache=False, ) if res: return res @@ -213,35 +220,94 @@ def _get_element( ) return None + def _try_cached_selectors( + self, page: "Page", soup: BeautifulSoup, prompt: str, only_one: bool + ) -> Union[Optional[Element], List[Element]]: + """ + Attempts to retrieve elements using cached selectors with exponential backoff. -def attempt_with_backoff( - obj: DendritePageProtocol, - prompt_or_elements: Union[str, Dict[str, str]], - only_one: bool, - remaining_timeout: float, - only_use_cache: bool = False, -) -> Union[Optional[Element], List[Element], ElementsResponse]: - TIMEOUT_INTERVAL: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0] + Args: + page: The current page object + soup: The BeautifulSoup object of the current page + prompt: The prompt to search for + only_one: Whether to return only one element + + Returns: + The found elements if successful, None otherwise + """ + dto = CachedSelectorDTO(url=page.url, prompt=prompt) + selectors = self._get_logic_api().get_cached_selectors(dto) + if len(selectors) == 0: + logger.debug("No cached selectors found") + return None + logger.debug("Attempting to use cached selectors with backoff") + str_selectors = list(map(lambda x: x.selector, selectors)) + + def try_cached_selectors(): + return get_elements_from_selectors_soup(page, soup, str_selectors, only_one) + + return _attempt_with_backoff_helper( + "cached_selectors", try_cached_selectors, timeout=CACHE_TIMEOUT + ) + + +def _attempt_with_backoff_helper( + operation_name: str, + operation: Callable, + timeout: float, + backoff_intervals: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0], +) -> Optional[Any]: + """ + Generic helper function that implements exponential backoff for operations. + + Args: + operation_name: Name of the operation for logging + operation: Async function to execute + timeout: Maximum time to spend attempting the operation + backoff_intervals: List of timeouts between attempts + + Returns: + The result of the operation if successful, None otherwise + """ total_elapsed_time = 0 start_time = time.time() - for current_timeout in TIMEOUT_INTERVAL: - if total_elapsed_time >= remaining_timeout: + for i, current_timeout in enumerate(backoff_intervals): + if total_elapsed_time >= timeout: logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") return None request_start_time = time.time() - page = obj._get_page() - page_information = page.get_page_information( - include_screenshot=not only_use_cache + result = operation() + request_duration = time.time() - request_start_time + if result: + return result + sleep_duration = max(0, current_timeout - request_duration) + logger.info( + f"{operation_name} attempt {i + 1} failed. Sleeping for {sleep_duration:.2f} seconds" ) + time.sleep(sleep_duration) + total_elapsed_time = time.time() - start_time + logger.error( + f"All {operation_name} attempts failed after {total_elapsed_time:.2f} seconds" + ) + return None + + +def try_get_element( + obj: DendritePageProtocol, + prompt_or_elements: Union[str, Dict[str, str]], + only_one: bool, + remaining_timeout: float, +) -> Union[Optional[Element], List[Element], ElementsResponse]: + + def _try_get_element(): + page = obj._get_page() + page_information = page.get_page_information() dto = GetElementsDTO( page_information=page_information, prompt=prompt_or_elements, - use_cache=only_use_cache, only_one=only_one, - force_use_cache=only_use_cache, ) res = obj._get_logic_api().get_element(dto) - request_duration = time.time() - request_start_time if res.status == "impossible": logger.error( f"Impossible to get elements for '{prompt_or_elements}'. Reason: {res.message}" @@ -249,38 +315,22 @@ def attempt_with_backoff( return None if res.status == "success": logger.success(f"d[id]: {res.d_id} Selectors:{res.selectors}") - response = get_elements_from_selectors_soup( - page, page._get_previous_soup(), res, only_one - ) - if response: - return response - sleep_duration = max(0, current_timeout - request_duration) - logger.info( - f"Failed to get elements for prompt:\n\n'{prompt_or_elements}'\n\nStatus: {res.status}\n\nMessage: {res.message}\n\nSleeping for {sleep_duration:.2f} seconds" - ) - time.sleep(sleep_duration) - total_elapsed_time = time.time() - start_time - logger.error(f"All attempts failed after {total_elapsed_time:.2f} seconds") - return None + if res.selectors is not None: + return get_elements_from_selectors_soup( + page, page._get_previous_soup(), res.selectors, only_one + ) + return None + return _attempt_with_backoff_helper( + "find_element_agent", _try_get_element, remaining_timeout + ) -def get_elements_from_selectors( - obj: DendritePageProtocol, res: GetElementResponse, only_one: bool -) -> Union[Optional[Element], List[Element], ElementsResponse]: - if isinstance(res.selectors, dict): - result = {} - for key, selectors in res.selectors.items(): - for selector in selectors: - page = obj._get_page() - dendrite_elements = page._get_all_elements_from_selector(selector) - if len(dendrite_elements) > 0: - result[key] = dendrite_elements[0] - break - return ElementsResponse(result) - elif isinstance(res.selectors, list): - for selector in reversed(res.selectors): - page = obj._get_page() - dendrite_elements = page._get_all_elements_from_selector(selector) - if len(dendrite_elements) > 0: - return dendrite_elements[0] if only_one else dendrite_elements + +def get_elements_from_selectors_soup( + page: "Page", soup: BeautifulSoup, selectors: List[str], only_one: bool +) -> Union[Optional[Element], List[Element]]: + for selector in reversed(selectors): + dendrite_elements = _get_all_elements_from_selector_soup(selector, soup, page) + if len(dendrite_elements) > 0: + return dendrite_elements[0] if only_one else dendrite_elements return None diff --git a/dendrite/logic/ask/ask.py b/dendrite/logic/ask/ask.py index 9f9ecff..25ea9c4 100644 --- a/dendrite/logic/ask/ask.py +++ b/dendrite/logic/ask/ask.py @@ -9,22 +9,20 @@ ) +from dendrite.logic.config import Config from dendrite.logic.llm.agent import Agent, Message -from dendrite.logic.llm.config import llm_config from dendrite.models.dto.ask_page_dto import AskPageDTO from dendrite.models.response.ask_page_response import AskPageResponse from .image import segment_image -async def ask_page_action( - ask_page_dto: AskPageDTO, -) -> AskPageResponse: +async def ask_page_action(ask_page_dto: AskPageDTO, config: Config) -> AskPageResponse: image_segments = segment_image( ask_page_dto.page_information.screenshot_base64, segment_height=2000 ) - agent = Agent(llm_config.get("ask_page_agent")) + agent = Agent(config.llm_config.get("ask_page_agent")) scrolled_to_segment_i = 0 content = generate_ask_page_prompt(ask_page_dto, image_segments) messages: List[Message] = [ diff --git a/dendrite/logic/config.py b/dendrite/logic/config.py index e2f88a9..4735e71 100644 --- a/dendrite/logic/config.py +++ b/dendrite/logic/config.py @@ -1,15 +1,18 @@ from pathlib import Path +from typing import Optional, Union from dendrite.logic.cache.file_cache import FileCache +from dendrite.logic.llm.config import LLMConfig from dendrite.models.scripts import Script from dendrite.models.selector import Selector class Config: - def __init__(self): - self.cache_path = Path("./cache") - self.llm_config = "8udjsad" + def __init__( + self, + cache_path: Optional[Union[str, Path]] = None, + llm_config: Optional[LLMConfig] = None, + ): + self.cache_path = Path(cache_path) if cache_path else Path("./.dendrite/cache") + self.llm_config = llm_config or LLMConfig() self.extract_cache = FileCache(Script, self.cache_path / "extract.json") self.element_cache = FileCache(Selector, self.cache_path / "get_element.json") - - -config = Config() diff --git a/dendrite/logic/extract/extract.py b/dendrite/logic/extract/extract.py index f37cfc4..708eb5c 100644 --- a/dendrite/logic/extract/extract.py +++ b/dendrite/logic/extract/extract.py @@ -5,6 +5,7 @@ from loguru import logger +from dendrite.logic.config import Config from dendrite.logic.extract.cached_script import get_working_cached_script from dendrite.logic.extract.extract_agent import ExtractAgent from dendrite.models.dto.extract_dto import ExtractDTO @@ -103,7 +104,7 @@ async def wait_for_notification( InMemoryLockManager.events.pop(self.key, None) -async def extract(extract_page_dto: ExtractDTO) -> ExtractResponse: +async def extract(extract_page_dto: ExtractDTO, config: Config) -> ExtractResponse: # Check cache usage flags if extract_page_dto.use_cache or extract_page_dto.force_use_cache: res = await test_cache(extract_page_dto) @@ -121,27 +122,23 @@ async def extract(extract_page_dto: ExtractDTO) -> ExtractResponse: lock_acquired = await lock_manager.acquire_lock() if lock_acquired: - return await generate_script(extract_page_dto, lock_manager) + return await generate_script(extract_page_dto, lock_manager, config) else: res = await wait_for_script_generation(extract_page_dto, lock_manager) if res: return res # Else create a working script since page is different - extract_agent = ExtractAgent( - extract_page_dto.page_information, - ) + extract_agent = ExtractAgent(extract_page_dto.page_information, config=config) res = await extract_agent.write_and_run_script(extract_page_dto) return res async def generate_script( - extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager + extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager, config: Config ) -> ExtractResponse: try: - extract_agent = ExtractAgent( - extract_page_dto.page_information, - ) + extract_agent = ExtractAgent(extract_page_dto.page_information, config=config) res = await extract_agent.write_and_run_script(extract_page_dto) await lock_manager.publish("done") return res diff --git a/dendrite/logic/extract/extract_agent.py b/dendrite/logic/extract/extract_agent.py index 4828d88..c374ac6 100644 --- a/dendrite/logic/extract/extract_agent.py +++ b/dendrite/logic/extract/extract_agent.py @@ -6,6 +6,7 @@ from loguru import logger from dendrite.logic.cache.utils import save_script +from dendrite.logic.config import Config from dendrite.logic.dom.strip import mild_strip from dendrite.logic.extract.prompts import ( LARGE_HTML_CHAR_TRUNCATE_LEN, @@ -25,20 +26,16 @@ from dendrite.models.response.extract_response import ExtractResponse from ..code.code_session import CodeSession from ..ask.image import segment_image -from ..llm.config import llm_config class ExtractAgent(Agent): - def __init__( - self, - page_information: PageInformation, - ) -> None: - super().__init__(llm_config.get("extract_agent")) + def __init__(self, page_information: PageInformation, config: Config) -> None: + super().__init__(config.llm_config.get("extract_agent")) self.page_information = page_information self.soup = BeautifulSoup(page_information.raw_html, "lxml") self.messages = [] self.generated_script: Optional[str] = None - self.llm_config = llm_config + self.config = config def get_generated_script(self): return self.generated_script @@ -48,13 +45,13 @@ async def write_and_run_script( ) -> ExtractResponse: mild_soup = mild_strip(self.soup) - search_terms = [] - segments = segment_image( extract_page_dto.page_information.screenshot_base64, segment_height=4000 ) - scroll_agent = ScrollAgent(self.page_information) + scroll_agent = ScrollAgent( + self.page_information, llm_config=self.config.llm_config + ) scroll_result = await scroll_agent.scroll_through_page( extract_page_dto.combined_prompt, image_segments=segments, @@ -80,8 +77,7 @@ async def write_and_run_script( + "\n- ".join(scroll_result.element_to_inspect_html) ) expanded = await get_expanded_dom( - mild_soup, - combined_prompt, + mild_soup, combined_prompt, self.config.llm_config ) if expanded: expanded_html = expanded[0] diff --git a/dendrite/logic/extract/scroll_agent.py b/dendrite/logic/extract/scroll_agent.py index 59fc914..ea18145 100644 --- a/dendrite/logic/extract/scroll_agent.py +++ b/dendrite/logic/extract/scroll_agent.py @@ -10,8 +10,8 @@ from loguru import logger from dendrite.logic.llm.agent import Agent, Message +from dendrite.logic.llm.config import LLMConfig from dendrite.models.page_information import PageInformation -from dendrite.logic.llm.config import llm_config ScrollActionStatus = Literal["done", "scroll_down", "loading", "error"] @@ -64,7 +64,7 @@ def parse(self, data_dict: dict, segment_i: int) -> Optional[ScrollResult]: class ScrollAgent(Agent): - def __init__(self, page_information: PageInformation): + def __init__(self, page_information: PageInformation, llm_config: LLMConfig): super().__init__(llm_config.get("scroll_agent")) self.page_information = page_information self.choices: List[ScrollRes] = [ diff --git a/dendrite/logic/get_element/agents/segment_agent.py b/dendrite/logic/get_element/agents/segment_agent.py index 8ebadc0..61c1d85 100644 --- a/dendrite/logic/get_element/agents/segment_agent.py +++ b/dendrite/logic/get_element/agents/segment_agent.py @@ -5,10 +5,8 @@ from annotated_types import Len from loguru import logger from pydantic import BaseModel, ValidationError - from dendrite.logic.llm.agent import Agent -from dendrite.logic.llm.config import llm_config - +from dendrite.logic.llm.config import LLMConfig from .prompts import SEGMENT_PROMPT @@ -83,9 +81,7 @@ def parse_segment_output(text: str, index: int) -> SegmentAgentReponseType: async def extract_relevant_d_ids( - prompt: str, - segments: List[str], - index: int, + prompt: str, segments: List[str], index: int, llm_config: LLMConfig ) -> SegmentAgentReponseType: agent = Agent(llm_config.get("segment_agent"), system_prompt=SEGMENT_PROMPT) message = "" diff --git a/dendrite/logic/get_element/agents/select_agent.py b/dendrite/logic/get_element/agents/select_agent.py index 658274c..1c2f6d0 100644 --- a/dendrite/logic/get_element/agents/select_agent.py +++ b/dendrite/logic/get_element/agents/select_agent.py @@ -7,7 +7,7 @@ from dendrite.browser._common.types import Status from dendrite.logic.llm.agent import Agent -from dendrite.logic.llm.config import llm_config +from dendrite.logic.llm.config import LLMConfig from ..hanifi_segment import SelectedTag from .prompts import SELECT_PROMPT @@ -24,6 +24,7 @@ async def select_best_tag( tags: List[SelectedTag], prompt: str, time_since_frame_navigated: Optional[float], + llm_config: LLMConfig, return_several: bool = False, ) -> Tuple[int, int, Optional[SelectAgentResponse]]: diff --git a/dendrite/logic/get_element/cached_selector.py b/dendrite/logic/get_element/cached_selector.py index 7c57c7c..07ad105 100644 --- a/dendrite/logic/get_element/cached_selector.py +++ b/dendrite/logic/get_element/cached_selector.py @@ -1,15 +1,9 @@ from datetime import datetime -from typing import Optional, Type +from typing import Optional from urllib.parse import urlparse -from bs4 import BeautifulSoup -from loguru import logger -from pydantic import BaseModel - - from dendrite.logic.cache.file_cache import FileCache from dendrite.models.selector import Selector -from dendrite.logic.config import config async def get_selector_from_cache( @@ -20,8 +14,9 @@ async def get_selector_from_cache( return cache.get({"netloc": netloc, "prompt": prompt}) -async def add_selector_to_cache(prompt: str, bs4_selector: str, url: str) -> None: - cache = config.element_cache +async def add_selector_to_cache( + prompt: str, bs4_selector: str, url: str, cache: FileCache[Selector] +) -> None: created_at = datetime.now().isoformat() netloc = urlparse(url).netloc selector: Selector = Selector( diff --git a/dendrite/logic/get_element/get_element.py b/dendrite/logic/get_element/get_element.py index 10ba487..2df4497 100644 --- a/dendrite/logic/get_element/get_element.py +++ b/dendrite/logic/get_element/get_element.py @@ -1,58 +1,43 @@ -from typing import Optional +from typing import List, Optional from bs4 import BeautifulSoup, Tag from loguru import logger -from dendrite.logic.config import config +from dendrite.logic.config import Config from dendrite.logic.dom.css import check_if_selector_successful, find_css_selector from dendrite.logic.dom.strip import remove_hidden_elements from dendrite.logic.get_element.cached_selector import ( add_selector_to_cache, get_selector_from_cache, ) +from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO from dendrite.models.dto.get_elements_dto import GetElementsDTO from dendrite.models.response.get_element_response import GetElementResponse +from dendrite.models.selector import Selector from .hanifi_search import hanifi_search -async def get_element(dto: GetElementsDTO) -> GetElementResponse: +async def get_element(dto: GetElementsDTO, config: Config) -> GetElementResponse: if isinstance(dto.prompt, str): - return await process_prompt(dto.prompt, dto) + return await process_prompt(dto.prompt, dto, config) raise ... -async def process_prompt(prompt: str, dto: GetElementsDTO) -> GetElementResponse: - +async def process_prompt( + prompt: str, dto: GetElementsDTO, config: Config +) -> GetElementResponse: soup = BeautifulSoup(dto.page_information.raw_html, "lxml") - - if dto.use_cache: - res = await check_cache( - soup, - dto.page_information.url, - prompt, - dto.only_one, - ) - if res: - return res - - if dto.force_use_cache: - return GetElementResponse( - selectors=[], - status="failed", - message="Forced to use cache, but no cached selectors found", - used_cache=False, - ) - - return await get_new_element(soup, prompt, dto) + return await get_new_element(soup, prompt, dto, config) async def get_new_element( - soup: BeautifulSoup, prompt: str, dto: GetElementsDTO + soup: BeautifulSoup, prompt: str, dto: GetElementsDTO, config: Config ) -> GetElementResponse: soup_without_hidden_elements = remove_hidden_elements(soup) element = await hanifi_search( soup_without_hidden_elements, prompt, + config, dto.page_information.time_since_frame_navigated, ) interactable = element[0] @@ -64,17 +49,18 @@ async def get_new_element( tag = soup.find(attrs={"d-id": interactable.dendrite_id}) if isinstance(tag, Tag): selector = find_css_selector(tag, soup) + cache = config.element_cache await add_selector_to_cache( prompt, bs4_selector=selector, url=dto.page_information.url, + cache=cache, ) return GetElementResponse( selectors=[selector], message=interactable.reason, d_id=interactable.dendrite_id, status="success", - used_cache=False, ) interactable.status = "failed" interactable.reason = "d-id does not exist in the soup" @@ -82,27 +68,36 @@ async def get_new_element( return GetElementResponse( message=interactable.reason, status=interactable.status, - used_cache=False, ) -async def check_cache( - soup: BeautifulSoup, url: str, prompt: str, only_one: bool -) -> Optional[GetElementResponse]: - cache = config.element_cache - db_selectors = await get_selector_from_cache(url, prompt, cache) +async def get_cached_selector(dto: CachedSelectorDTO, config: Config) -> List[Selector]: + if not isinstance(dto.prompt, str): + return [] + db_selectors = await get_selector_from_cache( + dto.url, dto.prompt, config.element_cache + ) if db_selectors is None: - return None + return [] + + return [db_selectors] + + +# async def check_cache( +# soup: BeautifulSoup, url: str, prompt: str, only_one: bool, config: Config +# ) -> Optional[GetElementResponse]: +# cache = config.element_cache +# db_selectors = await get_selector_from_cache(url, prompt, cache) - successful_selectors = [] +# if db_selectors is None: +# return None - if check_if_selector_successful(db_selectors.selector, soup, only_one): - return GetElementResponse( - selectors=[db_selectors.selector], - status="success", - used_cache=True, - ) +# if check_if_selector_successful(db_selectors.selector, soup, only_one): +# return GetElementResponse( +# selectors=[db_selectors.selector], +# status="success", +# ) # async def get_cached_selector(dto: GetCachedSelectorDTO) -> Optional[Selector]: diff --git a/dendrite/logic/get_element/hanifi_search.py b/dendrite/logic/get_element/hanifi_search.py index b3406f2..be8d357 100644 --- a/dendrite/logic/get_element/hanifi_search.py +++ b/dendrite/logic/get_element/hanifi_search.py @@ -3,8 +3,9 @@ from bs4 import BeautifulSoup, Tag +from dendrite.logic.config import Config from dendrite.logic.dom.strip import strip_soup -from dendrite.logic.llm.config import llm_config +from dendrite.logic.llm.config import LLMConfig from .agents import segment_agent, select_agent from .agents.segment_agent import ( @@ -18,11 +19,11 @@ async def get_expanded_dom( - soup: BeautifulSoup, prompt: str + soup: BeautifulSoup, prompt: str, llm_config: LLMConfig ) -> Optional[Tuple[str, List[SegmentAgentReponseType], List[SelectedTag]]]: new_nodes = hanifi_segment(soup, 6000, 3) - tags = await get_relevant_tags(prompt, new_nodes) + tags = await get_relevant_tags(prompt, new_nodes, llm_config) succesful_d_ids = [ (tag.d_id, tag.index, tag.reason) @@ -48,12 +49,13 @@ async def get_expanded_dom( async def hanifi_search( soup: BeautifulSoup, prompt: str, + config: Config, time_since_frame_navigated: Optional[float] = None, return_several: bool = False, ) -> List[Element]: stripped_soup = strip_soup(soup) - expand_res = await get_expanded_dom(stripped_soup, prompt) + expand_res = await get_expanded_dom(stripped_soup, prompt, config.llm_config) if expand_res is None: return [Element(status="failed", reason="No element found when expanding HTML")] @@ -76,6 +78,7 @@ async def hanifi_search( flat_list, prompt, time_since_frame_navigated, + config.llm_config, return_several, ) @@ -99,12 +102,13 @@ async def hanifi_search( async def get_relevant_tags( prompt: str, segments: List[List[str]], + llm_config: LLMConfig, ) -> List[SegmentAgentReponseType]: tasks: List[Coroutine[Any, Any, SegmentAgentReponseType]] = [] for index, segment in enumerate(segments): - tasks.append(extract_relevant_d_ids(prompt, segment, index)) + tasks.append(extract_relevant_d_ids(prompt, segment, index, llm_config)) results: List[SegmentAgentReponseType] = await asyncio.gather(*tasks) if results is None: diff --git a/dendrite/logic/interfaces/async_api.py b/dendrite/logic/interfaces/async_api.py index 646b2eb..10c477a 100644 --- a/dendrite/logic/interfaces/async_api.py +++ b/dendrite/logic/interfaces/async_api.py @@ -1,10 +1,11 @@ -from typing import Protocol +from typing import List, Optional, Protocol -from dendrite.browser.async_api._core.models.authentication import AuthSession +from dendrite.logic.config import Config from dendrite.logic.get_element import get_element from dendrite.models.dto.ask_page_dto import AskPageDTO from dendrite.models.dto.extract_dto import ExtractDTO -from dendrite.models.dto.get_elements_dto import CheckSelectorCacheDTO, GetElementsDTO +from dendrite.models.dto.get_elements_dto import GetElementsDTO +from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO from dendrite.models.dto.make_interaction_dto import VerifyActionDTO from dendrite.models.response.ask_page_response import AskPageResponse @@ -15,6 +16,7 @@ from dendrite.logic.ask import ask from dendrite.logic.extract import extract from dendrite.logic import verify_interaction +from dendrite.models.selector import Selector class LogicAPIProtocol(Protocol): @@ -29,17 +31,21 @@ async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... class AsyncProtocol(LogicAPIProtocol): + + def __init__(self, config: Config): + self._config = config + async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: - return await get_element.get_element(dto) + return await get_element.get_element(dto, self._config) - # async def get_cached_selectors(self, dto: CheckSelectorCacheDTO) -> GetElementResponse: - # return await get_element.get_cached_selectors(dto) + async def get_cached_selectors(self, dto: CachedSelectorDTO) -> List[Selector]: + return await get_element.get_cached_selector(dto, self._config) async def extract(self, dto: ExtractDTO) -> ExtractResponse: - return await extract.extract(dto) + return await extract.extract(dto, self._config) async def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: - return await verify_interaction.verify_action(dto) + return await verify_interaction.verify_action(dto, self._config) async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: - return await ask.ask_page_action(dto) + return await ask.ask_page_action(dto, self._config) diff --git a/dendrite/logic/interfaces/sync_api.py b/dendrite/logic/interfaces/sync_api.py index 4b75fc2..c34bdb1 100644 --- a/dendrite/logic/interfaces/sync_api.py +++ b/dendrite/logic/interfaces/sync_api.py @@ -1,13 +1,14 @@ import asyncio from concurrent.futures import ThreadPoolExecutor import threading -from typing import Any, Coroutine, Protocol, TypeVar +from typing import Any, Coroutine, List, Protocol, TypeVar -from dendrite.browser.async_api._core.models.authentication import AuthSession +from dendrite.logic.config import Config from dendrite.logic.get_element import get_element from dendrite.models.dto.ask_page_dto import AskPageDTO +from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO from dendrite.models.dto.extract_dto import ExtractDTO -from dendrite.models.dto.get_elements_dto import CheckSelectorCacheDTO, GetElementsDTO +from dendrite.models.dto.get_elements_dto import GetElementsDTO from dendrite.models.dto.make_interaction_dto import VerifyActionDTO from dendrite.models.response.ask_page_response import AskPageResponse @@ -18,6 +19,7 @@ from dendrite.logic.ask import ask from dendrite.logic.extract import extract from dendrite.logic import verify_interaction +from dendrite.models.selector import Selector T = TypeVar("T") @@ -60,14 +62,21 @@ def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... class SyncProtocol(LogicAPIProtocol): + + def __init__(self, config: Config): + self._config = config + def get_element(self, dto: GetElementsDTO) -> GetElementResponse: - return run_coroutine_sync(get_element.get_element(dto)) + return run_coroutine_sync(get_element.get_element(dto, self._config)) + + def get_cached_selectors(self, dto: CachedSelectorDTO) -> List[Selector]: + return run_coroutine_sync(get_element.get_cached_selector(dto, self._config)) def extract(self, dto: ExtractDTO) -> ExtractResponse: - return run_coroutine_sync(extract.extract(dto)) + return run_coroutine_sync(extract.extract(dto, self._config)) def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: - return run_coroutine_sync(verify_interaction.verify_action(dto)) + return run_coroutine_sync(verify_interaction.verify_action(dto, self._config)) def ask_page(self, dto: AskPageDTO) -> AskPageResponse: - return run_coroutine_sync(ask.ask_page_action(dto)) + return run_coroutine_sync(ask.ask_page_action(dto, self._config)) diff --git a/dendrite/logic/llm/config.py b/dendrite/logic/llm/config.py index 4f70e16..576d4b0 100644 --- a/dendrite/logic/llm/config.py +++ b/dendrite/logic/llm/config.py @@ -2,11 +2,6 @@ from dendrite.logic.llm.agent import LLM -try: - import tomllib # type: ignore -except ModuleNotFoundError: - import tomli as tomllib # type: ignore # tomllib is only included standard lib for python 3.11+ - AGENTS = Literal[ "extract_agent", "scroll_agent", @@ -107,7 +102,3 @@ def get( return self.default_llm return None - - -# Create a single instance -llm_config = LLMConfig() diff --git a/dendrite/logic/verify_interaction.py b/dendrite/logic/verify_interaction.py index f697edb..4efe943 100644 --- a/dendrite/logic/verify_interaction.py +++ b/dendrite/logic/verify_interaction.py @@ -3,14 +3,14 @@ from bs4 import BeautifulSoup +from dendrite.logic.config import Config from dendrite.logic.llm.agent import LLM, Agent, Message -from dendrite.logic.llm.config import llm_config from dendrite.models.dto.make_interaction_dto import VerifyActionDTO from dendrite.models.response.interaction_response import InteractionResponse async def verify_action( - make_interaction_dto: VerifyActionDTO, + make_interaction_dto: VerifyActionDTO, config: Config ) -> InteractionResponse: if ( @@ -77,7 +77,7 @@ async def verify_action( ] default = LLM(model="gpt-4o", max_tokens=150) - llm = Agent(llm_config.get("verify_action", default)) + llm = Agent(config.llm_config.get("verify_action", default)) res = await llm.call_llm(messages) try: diff --git a/dendrite/models/dto/cached_selector_dto.py b/dendrite/models/dto/cached_selector_dto.py new file mode 100644 index 0000000..f4b243b --- /dev/null +++ b/dendrite/models/dto/cached_selector_dto.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class CachedSelectorDTO(BaseModel): + url: str + prompt: str diff --git a/dendrite/models/dto/get_elements_dto.py b/dendrite/models/dto/get_elements_dto.py index 95bb126..aeb488d 100644 --- a/dendrite/models/dto/get_elements_dto.py +++ b/dendrite/models/dto/get_elements_dto.py @@ -13,6 +13,4 @@ class CheckSelectorCacheDTO(BaseModel): class GetElementsDTO(BaseModel): prompt: Union[str, Dict[str, str]] page_information: PageInformation - use_cache: bool = True - force_use_cache: bool = False only_one: bool diff --git a/dendrite/models/response/get_element_response.py b/dendrite/models/response/get_element_response.py index 3491818..6f2534c 100644 --- a/dendrite/models/response/get_element_response.py +++ b/dendrite/models/response/get_element_response.py @@ -8,6 +8,5 @@ class GetElementResponse(BaseModel): status: Status d_id: Optional[str] = None - selectors: Optional[Union[List[str], Dict[str, List[str]]]] = None + selectors: Optional[List[str]] = None message: str = "" - used_cache: bool = False From e4980235a8db1564c9a4d55b1355817076eda5e6 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Tue, 3 Dec 2024 13:29:31 +0100 Subject: [PATCH 08/18] Add CachedExtractDTO and update extract logic for caching scripts - Introduced CachedExtractDTO for managing cached script data. - Updated get_script function to accept URL instead of domain. - Modified extract logic to include caching behavior for scripts. - Removed unused cache flags from ExtractDTO and adjusted related logic. --- .../browser/async_api/_core/mixin/extract.py | 194 +++++++++++++----- .../async_api/_core/protocol/page_protocol.py | 1 - .../browser/sync_api/_core/mixin/extract.py | 173 +++++++++++----- .../sync_api/_core/protocol/page_protocol.py | 1 - dendrite/logic/cache/utils.py | 3 +- dendrite/logic/code/code_session.py | 13 +- dendrite/logic/extract/cached_script.py | 9 +- dendrite/logic/extract/extract.py | 24 +-- dendrite/logic/hosted/_api/__init__.py | 0 dendrite/logic/hosted/_api/_http_client.py | 72 ------- .../logic/hosted/_api/browser_api_client.py | 52 ----- dendrite/logic/hosted/async_api_impl.py | 0 dendrite/logic/interfaces/async_api.py | 5 + dendrite/logic/interfaces/sync_api.py | 5 + dendrite/models/dto/cached_extract_dto.py | 6 + dendrite/models/dto/extract_dto.py | 19 -- dendrite/models/response/extract_response.py | 1 - 17 files changed, 304 insertions(+), 274 deletions(-) delete mode 100644 dendrite/logic/hosted/_api/__init__.py delete mode 100644 dendrite/logic/hosted/_api/_http_client.py delete mode 100644 dendrite/logic/hosted/_api/browser_api_client.py delete mode 100644 dendrite/logic/hosted/async_api_impl.py create mode 100644 dendrite/models/dto/cached_extract_dto.py diff --git a/dendrite/browser/async_api/_core/mixin/extract.py b/dendrite/browser/async_api/_core/mixin/extract.py index 0d1d5a1..cc3221b 100644 --- a/dendrite/browser/async_api/_core/mixin/extract.py +++ b/dendrite/browser/async_api/_core/mixin/extract.py @@ -1,6 +1,6 @@ import asyncio import time -from typing import Any, List, Optional, Type, overload +from typing import Any, Callable, List, Optional, Type, overload from loguru import logger @@ -15,8 +15,11 @@ to_json_schema, ) from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.logic.code.code_session import execute +from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.dto.extract_dto import ExtractDTO from dendrite.models.response.extract_response import ExtractResponse +from dendrite.models.scripts import Script CACHE_TIMEOUT = 5 @@ -115,7 +118,6 @@ async def extract( Raises: TimeoutError: If the extraction process exceeds the specified timeout. """ - logger.info(f"Starting extraction with prompt: {prompt}") json_schema = None @@ -131,28 +133,21 @@ async def extract( navigation_tracker = NavigationTracker(page) navigation_tracker.start_nav_tracking() - # Check if a script exists in the cache + # First try using cached extraction if enabled if use_cache: - logger.info("Cache available, attempting to use cached extraction") - result = await attempt_extraction_with_backoff( - self, - prompt, - json_schema, - remaining_timeout=CACHE_TIMEOUT, - only_use_cache=True, - ) - if result: - return convert_and_return_result(result, type_spec) + logger.info("Testing cache") + cached_result = await self._try_cached_extraction(prompt, json_schema) + if cached_result: + return convert_and_return_result(cached_result, type_spec) + # If cache failed or disabled, proceed with extraction agent logger.info( "Using extraction agent to perform extraction, since no cache was found or failed." ) - result = await attempt_extraction_with_backoff( - self, + result = await self._extract_with_agent( prompt, json_schema, - remaining_timeout=timeout - (time.time() - start_time), - only_use_cache=False, + timeout - (time.time() - start_time), ) if result: @@ -161,59 +156,141 @@ async def extract( logger.error(f"Extraction failed after {time.time() - start_time:.2f} seconds") return None + async def _try_cached_extraction( + self, + prompt: str, + json_schema: Optional[JsonSchema], + ) -> Optional[ExtractResponse]: + """ + Attempts to extract data using cached scripts with exponential backoff. -async def attempt_extraction_with_backoff( - obj: DendritePageProtocol, - prompt: str, - json_schema: Optional[JsonSchema], - remaining_timeout: float = 180.0, - only_use_cache: bool = False, -) -> Optional[ExtractResponse]: - TIMEOUT_INTERVAL: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0] - total_elapsed_time = 0 - start_time = time.time() + Args: + prompt: The prompt describing what to extract + json_schema: Optional JSON schema for type validation - for current_timeout in TIMEOUT_INTERVAL: - if total_elapsed_time >= remaining_timeout: - logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") + Returns: + ExtractResponse if successful, None otherwise + """ + page = await self._get_page() + dto = CachedExtractDTO(url=page.url, prompt=prompt) + scripts = await self._get_logic_api().get_cached_scripts(dto) + logger.debug(f"Found {len(scripts)} scripts in cache, {scripts}") + if len(scripts) == 0: + logger.debug( + f"No scripts found in cache for prompt: {prompt} in domain: {page.url}" + ) return None - request_start_time = time.time() - page = await obj._get_page() - page_information = await page.get_page_information( - include_screenshot=not only_use_cache + async def try_cached_extract(): + page = await self._get_page() + soup = await page._get_soup() + for script in scripts: + res = await test_script(script, str(soup), json_schema) + if res is not None: + return ExtractResponse( + status="success", + message="Re-used a preexisting script from cache with the same specifications.", + return_data=res, + created_script=script.script, + ) + + return None + + return await _attempt_with_backoff_helper( + "cached_extraction", + try_cached_extract, + CACHE_TIMEOUT, ) - extract_dto = ExtractDTO( - page_information=page_information, - prompt=prompt, - return_data_json_schema=json_schema, - use_screenshot=True, - use_cache=only_use_cache, - force_use_cache=only_use_cache, + + async def _extract_with_agent( + self, + prompt: str, + json_schema: Optional[JsonSchema], + remaining_timeout: float, + ) -> Optional[ExtractResponse]: + """ + Attempts to extract data using the extraction agent with exponential backoff. + + Args: + prompt: The prompt describing what to extract + json_schema: Optional JSON schema for type validation + remaining_timeout: Maximum time to spend on extraction + + Returns: + ExtractResponse if successful, None otherwise + """ + + async def try_extract_with_agent(): + page = await self._get_page() + page_information = await page.get_page_information(include_screenshot=True) + extract_dto = ExtractDTO( + page_information=page_information, + prompt=prompt, + return_data_json_schema=json_schema, + use_screenshot=True, + ) + + res: ExtractResponse = await self._get_logic_api().extract(extract_dto) + + if res.status == "impossible": + logger.error(f"Impossible to extract data. Reason: {res.message}") + return None + + if res.status == "success": + logger.success(f"Extraction successful: '{res.message}'") + return res + + return None + + return await _attempt_with_backoff_helper( + "extraction_agent", + try_extract_with_agent, + remaining_timeout, ) - res: ExtractResponse = await obj._get_logic_api().extract(extract_dto) - request_duration = time.time() - request_start_time - if res.status == "impossible": - logger.error(f"Impossible to extract data. Reason: {res.message}") +async def _attempt_with_backoff_helper( + operation_name: str, + operation: Callable, + timeout: float, + backoff_intervals: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0], +) -> Optional[Any]: + """ + Generic helper function that implements exponential backoff for operations. + + Args: + operation_name: Name of the operation for logging + operation: Async function to execute + timeout: Maximum time to spend attempting the operation + backoff_intervals: List of timeouts between attempts + + Returns: + The result of the operation if successful, None otherwise + """ + total_elapsed_time = 0 + start_time = time.time() + + for i, current_timeout in enumerate(backoff_intervals): + if total_elapsed_time >= timeout: + logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") return None - if res.status == "success": - logger.success( - f"Extraction successful: '{res.message}'\nUsed cache: {res.used_cache}" - ) - return res + request_start_time = time.time() + result = await operation() + request_duration = time.time() - request_start_time + + if result: + return result sleep_duration = max(0, current_timeout - request_duration) logger.info( - f"Extraction attempt failed. Status: {res.status}\nMessage: {res.message}\nSleeping for {sleep_duration:.2f} seconds" + f"{operation_name} attempt {i+1} failed. Sleeping for {sleep_duration:.2f} seconds" ) await asyncio.sleep(sleep_duration) total_elapsed_time = time.time() - start_time logger.error( - f"All extraction attempts failed after {total_elapsed_time:.2f} seconds" + f"All {operation_name} attempts failed after {total_elapsed_time:.2f} seconds" ) return None @@ -228,3 +305,16 @@ def convert_and_return_result( logger.info("Extraction process completed successfully") return converted_res + + +async def test_script( + script: Script, + raw_html: str, + return_data_json_schema: Any, +) -> Optional[Any]: + + try: + res = execute(script.script, raw_html, return_data_json_schema) + return res + except Exception as e: + logger.debug(f"Script failed with error: {str(e)} ") diff --git a/dendrite/browser/async_api/_core/protocol/page_protocol.py b/dendrite/browser/async_api/_core/protocol/page_protocol.py index afe51cd..b828af0 100644 --- a/dendrite/browser/async_api/_core/protocol/page_protocol.py +++ b/dendrite/browser/async_api/_core/protocol/page_protocol.py @@ -1,6 +1,5 @@ from typing import TYPE_CHECKING, Protocol -from dendrite.logic.hosted._api.browser_api_client import BrowserAPIClient from dendrite.logic.interfaces import AsyncProtocol if TYPE_CHECKING: diff --git a/dendrite/browser/sync_api/_core/mixin/extract.py b/dendrite/browser/sync_api/_core/mixin/extract.py index b6aa5ad..f704d4a 100644 --- a/dendrite/browser/sync_api/_core/mixin/extract.py +++ b/dendrite/browser/sync_api/_core/mixin/extract.py @@ -1,6 +1,6 @@ import time import time -from typing import Any, List, Optional, Type, overload +from typing import Any, Callable, List, Optional, Type, overload from loguru import logger from dendrite.browser.sync_api._core._managers.navigation_tracker import ( NavigationTracker, @@ -13,8 +13,11 @@ to_json_schema, ) from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.logic.code.code_session import execute +from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.dto.extract_dto import ExtractDTO from dendrite.models.response.extract_response import ExtractResponse +from dendrite.models.scripts import Script CACHE_TIMEOUT = 5 @@ -125,77 +128,137 @@ def extract( navigation_tracker = NavigationTracker(page) navigation_tracker.start_nav_tracking() if use_cache: - logger.info("Cache available, attempting to use cached extraction") - result = attempt_extraction_with_backoff( - self, - prompt, - json_schema, - remaining_timeout=CACHE_TIMEOUT, - only_use_cache=True, - ) - if result: - return convert_and_return_result(result, type_spec) + logger.info("Testing cache") + cached_result = self._try_cached_extraction(prompt, json_schema) + if cached_result: + return convert_and_return_result(cached_result, type_spec) logger.info( "Using extraction agent to perform extraction, since no cache was found or failed." ) - result = attempt_extraction_with_backoff( - self, - prompt, - json_schema, - remaining_timeout=timeout - (time.time() - start_time), - only_use_cache=False, + result = self._extract_with_agent( + prompt, json_schema, timeout - (time.time() - start_time) ) if result: return convert_and_return_result(result, type_spec) logger.error(f"Extraction failed after {time.time() - start_time:.2f} seconds") return None + def _try_cached_extraction( + self, prompt: str, json_schema: Optional[JsonSchema] + ) -> Optional[ExtractResponse]: + """ + Attempts to extract data using cached scripts with exponential backoff. + + Args: + prompt: The prompt describing what to extract + json_schema: Optional JSON schema for type validation + + Returns: + ExtractResponse if successful, None otherwise + """ + page = self._get_page() + dto = CachedExtractDTO(url=page.url, prompt=prompt) + scripts = self._get_logic_api().get_cached_scripts(dto) + logger.debug(f"Found {len(scripts)} scripts in cache, {scripts}") + if len(scripts) == 0: + logger.debug( + f"No scripts found in cache for prompt: {prompt} in domain: {page.url}" + ) + return None + + def try_cached_extract(): + page = self._get_page() + soup = page._get_soup() + for script in scripts: + res = test_script(script, str(soup), json_schema) + if res is not None: + return ExtractResponse( + status="success", + message="Re-used a preexisting script from cache with the same specifications.", + return_data=res, + created_script=script.script, + ) + return None + + return _attempt_with_backoff_helper( + "cached_extraction", try_cached_extract, CACHE_TIMEOUT + ) + + def _extract_with_agent( + self, prompt: str, json_schema: Optional[JsonSchema], remaining_timeout: float + ) -> Optional[ExtractResponse]: + """ + Attempts to extract data using the extraction agent with exponential backoff. + + Args: + prompt: The prompt describing what to extract + json_schema: Optional JSON schema for type validation + remaining_timeout: Maximum time to spend on extraction + + Returns: + ExtractResponse if successful, None otherwise + """ + + def try_extract_with_agent(): + page = self._get_page() + page_information = page.get_page_information(include_screenshot=True) + extract_dto = ExtractDTO( + page_information=page_information, + prompt=prompt, + return_data_json_schema=json_schema, + use_screenshot=True, + ) + res: ExtractResponse = self._get_logic_api().extract(extract_dto) + if res.status == "impossible": + logger.error(f"Impossible to extract data. Reason: {res.message}") + return None + if res.status == "success": + logger.success(f"Extraction successful: '{res.message}'") + return res + return None + + return _attempt_with_backoff_helper( + "extraction_agent", try_extract_with_agent, remaining_timeout + ) + + +def _attempt_with_backoff_helper( + operation_name: str, + operation: Callable, + timeout: float, + backoff_intervals: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0], +) -> Optional[Any]: + """ + Generic helper function that implements exponential backoff for operations. -def attempt_extraction_with_backoff( - obj: DendritePageProtocol, - prompt: str, - json_schema: Optional[JsonSchema], - remaining_timeout: float = 180.0, - only_use_cache: bool = False, -) -> Optional[ExtractResponse]: - TIMEOUT_INTERVAL: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0] + Args: + operation_name: Name of the operation for logging + operation: Async function to execute + timeout: Maximum time to spend attempting the operation + backoff_intervals: List of timeouts between attempts + + Returns: + The result of the operation if successful, None otherwise + """ total_elapsed_time = 0 start_time = time.time() - for current_timeout in TIMEOUT_INTERVAL: - if total_elapsed_time >= remaining_timeout: + for i, current_timeout in enumerate(backoff_intervals): + if total_elapsed_time >= timeout: logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") return None request_start_time = time.time() - page = obj._get_page() - page_information = page.get_page_information( - include_screenshot=not only_use_cache - ) - extract_dto = ExtractDTO( - page_information=page_information, - prompt=prompt, - return_data_json_schema=json_schema, - use_screenshot=True, - use_cache=only_use_cache, - force_use_cache=only_use_cache, - ) - res: ExtractResponse = obj._get_logic_api().extract(extract_dto) + result = operation() request_duration = time.time() - request_start_time - if res.status == "impossible": - logger.error(f"Impossible to extract data. Reason: {res.message}") - return None - if res.status == "success": - logger.success( - f"Extraction successful: '{res.message}'\nUsed cache: {res.used_cache}" - ) - return res + if result: + return result sleep_duration = max(0, current_timeout - request_duration) logger.info( - f"Extraction attempt failed. Status: {res.status}\nMessage: {res.message}\nSleeping for {sleep_duration:.2f} seconds" + f"{operation_name} attempt {i + 1} failed. Sleeping for {sleep_duration:.2f} seconds" ) time.sleep(sleep_duration) total_elapsed_time = time.time() - start_time logger.error( - f"All extraction attempts failed after {total_elapsed_time:.2f} seconds" + f"All {operation_name} attempts failed after {total_elapsed_time:.2f} seconds" ) return None @@ -209,3 +272,13 @@ def convert_and_return_result( converted_res = convert_to_type_spec(type_spec, res.return_data) logger.info("Extraction process completed successfully") return converted_res + + +def test_script( + script: Script, raw_html: str, return_data_json_schema: Any +) -> Optional[Any]: + try: + res = execute(script.script, raw_html, return_data_json_schema) + return res + except Exception as e: + logger.debug(f"Script failed with error: {str(e)} ") diff --git a/dendrite/browser/sync_api/_core/protocol/page_protocol.py b/dendrite/browser/sync_api/_core/protocol/page_protocol.py index f74afff..b37969e 100644 --- a/dendrite/browser/sync_api/_core/protocol/page_protocol.py +++ b/dendrite/browser/sync_api/_core/protocol/page_protocol.py @@ -1,5 +1,4 @@ from typing import TYPE_CHECKING, Protocol -from dendrite.logic.hosted._api.browser_api_client import BrowserAPIClient from dendrite.logic.interfaces import SyncProtocol if TYPE_CHECKING: diff --git a/dendrite/logic/cache/utils.py b/dendrite/logic/cache/utils.py index a3243b5..e5796fb 100644 --- a/dendrite/logic/cache/utils.py +++ b/dendrite/logic/cache/utils.py @@ -14,5 +14,6 @@ def save_script(code: str, prompt: str, url: str): extract_cache.ExtractCache.set({"prompt": prompt, "domain": domain}, script) -def get_script(prompt: str, domain: str) -> Optional[Script]: +def get_script(prompt: str, url: str) -> Optional[Script]: + domain = urlparse(url).netloc return extract_cache.ExtractCache.get({"prompt": prompt, "domain": domain}) diff --git a/dendrite/logic/code/code_session.py b/dendrite/logic/code/code_session.py index 6d2b4d4..fcb7300 100644 --- a/dendrite/logic/code/code_session.py +++ b/dendrite/logic/code/code_session.py @@ -120,11 +120,16 @@ def llm_readable_exec_res( show_length = 600 if var_name == "response_data" else 300 try: - # Convert var_value to string, handling potential errors - str_value = str(var_value) if var_value is not None else "None" + if var_value is None: + str_value = "None" + else: + str_value = str(var_value) + except Exception as e: - logger.error(f"Error converting to string for display: {e}") - str_value = f"" + logger.error( + f"Error converting to string for display: {e},\nvar_name: {var_name} | var_value{var_value}" + ) + str_value = "" truncated = truncate_long_string( str_value, max_len_end=show_length, max_len_start=show_length diff --git a/dendrite/logic/extract/cached_script.py b/dendrite/logic/extract/cached_script.py index 7420186..8164b29 100644 --- a/dendrite/logic/extract/cached_script.py +++ b/dendrite/logic/extract/cached_script.py @@ -14,14 +14,13 @@ async def get_working_cached_script( url: str, return_data_json_schema: Any, ) -> Optional[Tuple[Script, Any]]: - domain = urlparse(url).netloc if len(url) == 0: raise Exception("Domain must be specified") - scripts: List[Script] = [get_script(prompt, domain) or ...] + scripts: List[Script] = [get_script(prompt, url) or ...] logger.debug( - f"Found {len(scripts)} scripts in cache | Prompt: {prompt} in domain: {domain}" + f"Found {len(scripts)} scripts in cache | Prompt: {prompt} in domain: {url}" ) for script in scripts: @@ -30,7 +29,7 @@ async def get_working_cached_script( return script, res except Exception as e: logger.debug( - f"Script failed with error: {str(e)} | Prompt: {prompt} in domain: {domain}" + f"Script failed with error: {str(e)} | Prompt: {prompt} in domain: {url}" ) continue @@ -38,5 +37,5 @@ async def get_working_cached_script( return None raise Exception( - f"No working script found in cache even though {len(scripts)} scripts were available | Prompt: '{prompt}' in domain: '{domain}'" + f"No working script found in cache even though {len(scripts)} scripts were available | Prompt: '{prompt}' in domain: '{url}'" ) diff --git a/dendrite/logic/extract/extract.py b/dendrite/logic/extract/extract.py index 708eb5c..2196bfe 100644 --- a/dendrite/logic/extract/extract.py +++ b/dendrite/logic/extract/extract.py @@ -1,18 +1,23 @@ import asyncio import hashlib -from typing import Optional +from typing import List, Optional from urllib.parse import urlparse from loguru import logger +from dendrite.logic.cache.utils import get_script from dendrite.logic.config import Config from dendrite.logic.extract.cached_script import get_working_cached_script from dendrite.logic.extract.extract_agent import ExtractAgent +from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.dto.extract_dto import ExtractDTO from dendrite.models.response.extract_response import ExtractResponse +from dendrite.models.scripts import Script -# Assuming you have these imports -# from your_module import WebScrapingAgent, run_script_if_cached + +async def get_cached_scripts(dto: CachedExtractDTO, config: Config) -> List[Script]: + script = get_script(dto.prompt, dto.url) + return [script] if script else [] async def test_cache(extract_dto: ExtractDTO) -> Optional[ExtractResponse]: @@ -33,7 +38,6 @@ async def test_cache(extract_dto: ExtractDTO) -> Optional[ExtractResponse]: status="success", message="Re-used a preexisting script from cache with the same specifications.", return_data=script_exec_res, - used_cache=True, created_script=script.script, ) @@ -105,19 +109,7 @@ async def wait_for_notification( async def extract(extract_page_dto: ExtractDTO, config: Config) -> ExtractResponse: - # Check cache usage flags - if extract_page_dto.use_cache or extract_page_dto.force_use_cache: - res = await test_cache(extract_page_dto) - if res: - return res - - if extract_page_dto.force_use_cache: - return ExtractResponse( - status="failed", - message="No script available in cache that matches this prompt.", - ) - # Proceed with lock acquisition and processing lock_manager = InMemoryLockManager(extract_page_dto) lock_acquired = await lock_manager.acquire_lock() diff --git a/dendrite/logic/hosted/_api/__init__.py b/dendrite/logic/hosted/_api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/logic/hosted/_api/_http_client.py b/dendrite/logic/hosted/_api/_http_client.py deleted file mode 100644 index 69e0375..0000000 --- a/dendrite/logic/hosted/_api/_http_client.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -from typing import Any, Dict, Optional - -import httpx -from loguru import logger - -from dendrite.models.api_config import APIConfig - - -class HTTPClient: - def __init__(self, api_config: APIConfig, session_id: Optional[str] = None): - self.api_key = api_config.dendrite_api_key - self.api_config = api_config - self.session_id = session_id - self.base_url = self.resolve_base_url() - - def resolve_base_url(self): - base_url = ( - "http://localhost:8000/api/v1" - if os.environ.get("DENDRITE_DEV") - else "https://dendrite-server.azurewebsites.net/api/v1" - ) - return base_url - - async def send_request( - self, - endpoint: str, - params: Optional[dict] = None, - data: Optional[Dict[str, Any]] = None, - headers: Optional[dict] = None, - method: str = "GET", - ) -> httpx.Response: - url = f"{self.base_url}/{endpoint}" - - headers = headers or {} - headers["Content-Type"] = "application/json" - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - if self.session_id: - headers["X-Session-ID"] = self.session_id - - async with httpx.AsyncClient(timeout=300) as client: - try: - - # inject api_config to data - if data: - data["api_config"] = self.api_config.model_dump() - - response = await client.request( - method, url, params=params, json=data, headers=headers - ) - response.raise_for_status() - # logger.debug( - # f"{method} to '{url}', that took: { time.time() - start_time }\n\nResponse: {dict_res}\n\n" - # ) - return response - except httpx.HTTPStatusError as http_err: - logger.debug( - f"HTTP error occurred: {http_err.response.status_code}: {http_err.response.text}" - ) - raise - except httpx.ConnectError as connect_err: - logger.error( - f"Connection error occurred: {connect_err}. {url} Server might be down" - ) - raise - except httpx.RequestError as req_err: - # logger.debug(f"Request error occurred: {req_err}") - raise - except Exception as err: - # logger.debug(f"An error occurred: {err}") - raise diff --git a/dendrite/logic/hosted/_api/browser_api_client.py b/dendrite/logic/hosted/_api/browser_api_client.py deleted file mode 100644 index d9e6220..0000000 --- a/dendrite/logic/hosted/_api/browser_api_client.py +++ /dev/null @@ -1,52 +0,0 @@ -from dendrite.browser.async_api._core.models.authentication import AuthSession -from dendrite.logic.hosted._api._http_client import HTTPClient -from dendrite.models.dto.ask_page_dto import AskPageDTO -from dendrite.models.dto.extract_dto import ExtractDTO -from dendrite.models.dto.get_elements_dto import GetElementsDTO -from dendrite.models.dto.make_interaction_dto import VerifyActionDTO -from dendrite.models.response.ask_page_response import AskPageResponse -from dendrite.models.response.extract_response import ExtractResponse -from dendrite.models.response.get_element_response import GetElementResponse -from dendrite.models.response.interaction_response import InteractionResponse - - -class BrowserAPIClient(HTTPClient): - - async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: - res = await self.send_request( - "actions/get-interaction-selector", data=dto.model_dump(), method="POST" - ) - return GetElementResponse(**res.json()) - - async def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: - res = await self.send_request( - "actions/make-interaction", data=dto.model_dump(), method="POST" - ) - res_dict = res.json() - return InteractionResponse( - status=res_dict["status"], message=res_dict["message"] - ) - - async def extract(self, dto: ExtractDTO) -> ExtractResponse: - res = await self.send_request( - "actions/extract-page", data=dto.model_dump(), method="POST" - ) - res_dict = res.json() - return ExtractResponse( - status=res_dict["status"], - message=res_dict["message"], - return_data=res_dict["return_data"], - created_script=res_dict.get("created_script", None), - used_cache=res_dict.get("used_cache", False), - ) - - async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: - res = await self.send_request( - "actions/ask-page", data=dto.model_dump(), method="POST" - ) - res_dict = res.json() - return AskPageResponse( - status=res_dict["status"], - description=res_dict["description"], - return_data=res_dict["return_data"], - ) diff --git a/dendrite/logic/hosted/async_api_impl.py b/dendrite/logic/hosted/async_api_impl.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/logic/interfaces/async_api.py b/dendrite/logic/interfaces/async_api.py index 10c477a..5916ab8 100644 --- a/dendrite/logic/interfaces/async_api.py +++ b/dendrite/logic/interfaces/async_api.py @@ -3,6 +3,7 @@ from dendrite.logic.config import Config from dendrite.logic.get_element import get_element from dendrite.models.dto.ask_page_dto import AskPageDTO +from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.dto.extract_dto import ExtractDTO from dendrite.models.dto.get_elements_dto import GetElementsDTO from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO @@ -16,6 +17,7 @@ from dendrite.logic.ask import ask from dendrite.logic.extract import extract from dendrite.logic import verify_interaction +from dendrite.models.scripts import Script from dendrite.models.selector import Selector @@ -41,6 +43,9 @@ async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: async def get_cached_selectors(self, dto: CachedSelectorDTO) -> List[Selector]: return await get_element.get_cached_selector(dto, self._config) + async def get_cached_scripts(self, dto: CachedExtractDTO) -> List[Script]: + return await extract.get_cached_scripts(dto, self._config) + async def extract(self, dto: ExtractDTO) -> ExtractResponse: return await extract.extract(dto, self._config) diff --git a/dendrite/logic/interfaces/sync_api.py b/dendrite/logic/interfaces/sync_api.py index c34bdb1..0537b6f 100644 --- a/dendrite/logic/interfaces/sync_api.py +++ b/dendrite/logic/interfaces/sync_api.py @@ -6,6 +6,7 @@ from dendrite.logic.config import Config from dendrite.logic.get_element import get_element from dendrite.models.dto.ask_page_dto import AskPageDTO +from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO from dendrite.models.dto.extract_dto import ExtractDTO from dendrite.models.dto.get_elements_dto import GetElementsDTO @@ -19,6 +20,7 @@ from dendrite.logic.ask import ask from dendrite.logic.extract import extract from dendrite.logic import verify_interaction +from dendrite.models.scripts import Script from dendrite.models.selector import Selector @@ -72,6 +74,9 @@ def get_element(self, dto: GetElementsDTO) -> GetElementResponse: def get_cached_selectors(self, dto: CachedSelectorDTO) -> List[Selector]: return run_coroutine_sync(get_element.get_cached_selector(dto, self._config)) + def get_cached_scripts(self, dto: CachedExtractDTO) -> List[Script]: + return run_coroutine_sync(extract.get_cached_scripts(dto, self._config)) + def extract(self, dto: ExtractDTO) -> ExtractResponse: return run_coroutine_sync(extract.extract(dto, self._config)) diff --git a/dendrite/models/dto/cached_extract_dto.py b/dendrite/models/dto/cached_extract_dto.py new file mode 100644 index 0000000..83c4f47 --- /dev/null +++ b/dendrite/models/dto/cached_extract_dto.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class CachedExtractDTO(BaseModel): + url: str + prompt: str diff --git a/dendrite/models/dto/extract_dto.py b/dendrite/models/dto/extract_dto.py index 0bb4c74..e87544c 100644 --- a/dendrite/models/dto/extract_dto.py +++ b/dendrite/models/dto/extract_dto.py @@ -12,8 +12,6 @@ class ExtractDTO(BaseModel): return_data_json_schema: Any use_screenshot: bool = False - use_cache: bool = True - force_use_cache: bool = False @property def combined_prompt(self) -> str: @@ -24,20 +22,3 @@ def combined_prompt(self) -> str: else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" ) return f"Task: {self.prompt}{json_schema_prompt}" - - -class TryRunScriptDTO(BaseModel): - url: str - raw_html: str - prompt: str - db_prompt: Optional[str] = None - return_data_json_schema: Any - - @property - def combined_prompt(self) -> str: - json_schema_prompt = ( - "" - if self.return_data_json_schema == None - else f"\nJson schema: {json.dumps(self.return_data_json_schema)}" - ) - return f"Task: {self.prompt}{json_schema_prompt}" diff --git a/dendrite/models/response/extract_response.py b/dendrite/models/response/extract_response.py index 168e90a..87dba47 100644 --- a/dendrite/models/response/extract_response.py +++ b/dendrite/models/response/extract_response.py @@ -11,5 +11,4 @@ class ExtractResponse(BaseModel, Generic[T]): status: Status message: str return_data: Optional[T] = None - used_cache: bool = False created_script: Optional[str] = None From 32313137c4806189d9a14094a6afc273307f1284 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Tue, 3 Dec 2024 15:38:02 +0100 Subject: [PATCH 09/18] refactor extract agent and fix bug when requesting more html --- dendrite/logic/extract/extract_agent.py | 282 +++++++++++++----------- dendrite/logic/llm/agent.py | 5 +- 2 files changed, 152 insertions(+), 135 deletions(-) diff --git a/dendrite/logic/extract/extract_agent.py b/dendrite/logic/extract/extract_agent.py index c374ac6..03e894f 100644 --- a/dendrite/logic/extract/extract_agent.py +++ b/dendrite/logic/extract/extract_agent.py @@ -1,7 +1,7 @@ import json import re import sys -from typing import Any, List, Optional +from typing import List, Union from loguru import logger @@ -34,12 +34,9 @@ def __init__(self, page_information: PageInformation, config: Config) -> None: self.page_information = page_information self.soup = BeautifulSoup(page_information.raw_html, "lxml") self.messages = [] - self.generated_script: Optional[str] = None + self.current_segment = 0 self.config = config - def get_generated_script(self): - return self.generated_script - async def write_and_run_script( self, extract_page_dto: ExtractDTO ) -> ExtractResponse: @@ -84,7 +81,7 @@ async def write_and_run_script( if expanded_html: return await self.code_script_from_found_expanded_html_tags( - extract_page_dto, expanded_html, segments + extract_page_dto, expanded_html ) raise Exception("Failed to extract data from the page") # TODO: skriv bättre @@ -107,7 +104,7 @@ def segment_large_tag(self, tag): return segments async def code_script_from_found_expanded_html_tags( - self, extract_page_dto: ExtractDTO, expanded_html, segments + self, extract_page_dto: ExtractDTO, expanded_html ): agent_logger = logger.bind( @@ -137,11 +134,7 @@ async def code_script_from_found_expanded_html_tags( iterations = 0 max_retries = 10 - generated_script: str = "" - response_data: Any | None = None - - while iterations <= max_retries: - iterations += 1 + for iterations in range(max_retries): agent_logger.debug(f"Code generation | Iteration: {iterations}") text = await self.call_llm(messages) @@ -150,133 +143,154 @@ async def code_script_from_found_expanded_html_tags( json_pattern = r"```json(.*?)```" code_pattern = r"```python(.*?)```" - if text: - json_matches = re.findall(json_pattern, text, re.DOTALL) - code_matches = re.findall(code_pattern, text, re.DOTALL) - - if len(json_matches) + len(code_matches) > 1: - content = "Error: Please output only one action at a time (either JSON or Python code, not both)." - messages.append({"role": "user", "content": content}) + if text is None: + content = "Error: Failed to generate content." + messages.append({"role": "user", "content": content}) + continue + + json_matches = re.findall(json_pattern, text, re.DOTALL) + code_matches = re.findall(code_pattern, text, re.DOTALL) + + if len(json_matches) + len(code_matches) > 1: + content = "Error: Please output only one action at a time (either JSON or Python code, not both)." + messages.append({"role": "user", "content": content}) + continue + + if code_matches: + self.generated_script = code_matches[0].strip() + result = await self._handle_code_match( + code_matches[0].strip(), + messages, + iterations, + max_retries, + extract_page_dto, + agent_logger, + ) + + messages.extend(result) + continue + + elif json_matches: + result = self._handle_json_match(json_matches[0], expanded_html) + if isinstance(result, ExtractResponse): + save_script( + self.generated_script, + extract_page_dto.combined_prompt, + self.page_information.url, + ) + return result + elif isinstance(result, list): + messages.extend(result) continue - - for code_match in code_matches: - # agent_logger.debug("Processing code match") - generated_script = code_match.strip() - temp_code_session = CodeSession() - try: - variables = temp_code_session.exec_code( - generated_script, - self.soup, - self.page_information.raw_html, - ) - # agent_logger.debug("Code execution successful") - except Exception as e: - # agent_logger.error(f"Code execution failed: {str(e)}") - content = f"Error: {str(e)}" - messages.append({"role": "user", "content": content}) - continue - - try: - if "response_data" in variables: - response_data = variables["response_data"] - # agent_logger.debug(f"Response data: {response_data}") - - if extract_page_dto.return_data_json_schema != None: - temp_code_session.validate_response( - extract_page_dto.return_data_json_schema, - response_data, - ) - - llm_readable_exec_res = ( - temp_code_session.llm_readable_exec_res( - variables, - extract_page_dto.combined_prompt, - iterations, - max_retries, - ) - ) - - messages.append( - {"role": "user", "content": llm_readable_exec_res} - ) - continue - else: - content = ( - f"Error: You need to add the variable 'response_data'" - ) - messages.append( - { - "role": "user", - "content": content, - } - ) - continue - except Exception as e: - llm_readable_exec_res = temp_code_session.llm_readable_exec_res( - variables, - extract_page_dto.combined_prompt, - iterations, - max_retries, - ) - content = f"Error: Failed to validate `response_data`. Exception: {e}. {llm_readable_exec_res}" - messages.append( - { - "role": "user", - "content": content, - } - ) - continue - - for json_match in json_matches: - # agent_logger.debug("Processing JSON match") - extracted_json = json_match.strip() - data_dict = json.loads(extracted_json) - current_segment = 0 - if "request_more_html" in data_dict: - # agent_logger.info("Processing element indexes") - try: - current_segment += 1 - content = f"""Here is more of the HTML:\n```html\n{expanded_html[LARGE_HTML_CHAR_TRUNCATE_LEN*current_segment:LARGE_HTML_CHAR_TRUNCATE_LEN*(current_segment+1)]}\n```""" - if len(expanded_html) > LARGE_HTML_CHAR_TRUNCATE_LEN * ( - current_segment + 1 - ): - content += "\nThere is still more HTML to see. You can request more if needed." - else: - content += "\nThis is the end of the HTML content." - messages.append({"role": "user", "content": content}) - continue - except Exception as e: - # agent_logger.error( - # f"Error processing element indexes: {str(e)}" - # ) - content = f"Error: {str(e)}" - messages.append({"role": "user", "content": content}) - continue - elif "error" in data_dict: - # agent_logger.error(f"Error in data_dict: {data_dict['error']}") - raise Exception(data_dict["error"]) - elif "success" in data_dict: - # agent_logger.info("Script generation successful") - - self.generated_script = generated_script - save_script( - self.generated_script, - extract_page_dto.combined_prompt, - self.page_information.url, - ) - - # agent_logger.debug(f"Response data: {response_data}") - return ExtractResponse( - status="success", - message=data_dict["success"], - return_data=response_data, - created_script=self.get_generated_script(), - ) + else: + # If neither code nor json matches found, send error message + content = "Error: Could not find valid code or JSON in the assistant's response." + messages.append({"role": "user", "content": content}) + continue # agent_logger.warning("Failed to create script after retrying several times") return ExtractResponse( status="failed", message="Failed to create script after retrying several times.", return_data=None, - created_script=self.get_generated_script(), + created_script=self.generated_script, + ) + + async def _handle_code_match( + self, + generated_script: str, + messages: List[Message], + iterations, + max_retries, + extract_page_dto: ExtractDTO, + agent_logger, + ) -> List[Message]: + temp_code_session = CodeSession() + + try: + variables = temp_code_session.exec_code( + generated_script, self.soup, self.page_information.raw_html + ) + + if "response_data" not in variables: + return [ + { + "role": "user", + "content": "Error: You need to add the variable 'response_data'", + } + ] + + self.response_data = variables["response_data"] + + if extract_page_dto.return_data_json_schema: + temp_code_session.validate_response( + extract_page_dto.return_data_json_schema, self.response_data + ) + + llm_readable_exec_res = temp_code_session.llm_readable_exec_res( + variables, + extract_page_dto.combined_prompt, + iterations, + max_retries, + ) + + return [{"role": "user", "content": llm_readable_exec_res}] + + except Exception as e: + return [{"role": "user", "content": f"Error: {str(e)}"}] + + def _handle_json_match( + self, json_str: str, expanded_html: str + ) -> Union[ExtractResponse, List[Message]]: + try: + data_dict = json.loads(json_str) + + if "request_more_html" in data_dict: + return self._handle_more_html_request(expanded_html) + + if "error" in data_dict: + raise Exception(data_dict["error"]) + + if "success" in data_dict: + return ExtractResponse( + status="success", + message=data_dict["success"], + return_data=self.response_data, + created_script=self.generated_script, + ) + return [ + { + "role": "user", + "content": "Error: JSON response does not specify a valid action.", + } + ] + + except Exception as e: + return [{"role": "user", "content": f"Error: {str(e)}"}] + + def _handle_more_html_request(self, expanded_html: str) -> List[Message]: + + if LARGE_HTML_CHAR_TRUNCATE_LEN * (self.current_segment + 1) >= len( + expanded_html + ): + return [{"role": "user", "content": "There is no more HTML to show."}] + + self.current_segment += 1 + start = LARGE_HTML_CHAR_TRUNCATE_LEN * self.current_segment + end = min( + LARGE_HTML_CHAR_TRUNCATE_LEN * (self.current_segment + 1), + len(expanded_html), + ) + + content = ( + f"""Here is more of the HTML:\n```html\n{expanded_html[start:end]}\n```""" ) + + if len(expanded_html) > end: + content += ( + "\nThere is still more HTML to see. You can request more if needed." + ) + else: + content += "\nThis is the end of the HTML content." + + return [{"role": "user", "content": content}] diff --git a/dendrite/logic/llm/agent.py b/dendrite/logic/llm/agent.py index 5c179e7..4f8c2cf 100644 --- a/dendrite/logic/llm/agent.py +++ b/dendrite/logic/llm/agent.py @@ -1,3 +1,4 @@ +import json from typing import Any, Dict, List, Optional, Union, cast import litellm @@ -228,6 +229,8 @@ async def call_llm(self, messages: List[Message]) -> str: text = choices[0].message.content if text is None: - logger.error("No text content in the response") + logger.error( + f"No text content in the response | response: {res} ", + ) raise Exception("No text content in the response") return text From 68a1d91008ee5f7307ce791f9aac3afa77c8c73e Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Wed, 4 Dec 2024 16:01:13 +0100 Subject: [PATCH 10/18] add test implemenation of setup_auth --- dendrite/_cli/main.py | 27 +- .../browser/async_api/_core/_impl_browser.py | 34 +- .../async_api/_core/_local_browser_impl.py | 75 +++- .../async_api/_core/_managers/page_manager.py | 15 +- dendrite/browser/async_api/_core/_utils.py | 33 ++ .../async_api/_core/dendrite_browser.py | 113 ++++- .../browser/async_api/_core/dendrite_page.py | 12 +- dendrite/browser/async_api/_core/mixin/ask.py | 2 +- .../browser/async_api/_core/mixin/extract.py | 4 +- .../async_api/_core/mixin/get_element.py | 5 +- .../async_api/_core/models/authentication.py | 48 --- .../_core/models/page_diff_information.py | 0 .../_core/models/page_information.py | 16 - .../async_api/_core/protocol/page_protocol.py | 6 +- dendrite/browser/sync_api/__init__.py | 7 - dendrite/browser/sync_api/_core/__init__.py | 0 .../browser/sync_api/_core/_impl_browser.py | 61 --- .../browser/sync_api/_core/_impl_mapping.py | 29 -- .../browser/sync_api/_core/_js/__init__.py | 11 - .../sync_api/_core/_js/eventListenerPatch.js | 90 ---- .../sync_api/_core/_js/generateDendriteIDs.js | 88 ---- .../_core/_js/generateDendriteIDsIframe.js | 93 ----- .../sync_api/_core/_local_browser_impl.py | 27 -- .../sync_api/_core/_managers/__init__.py | 0 .../_core/_managers/navigation_tracker.py | 67 --- .../sync_api/_core/_managers/page_manager.py | 74 ---- .../_core/_managers/screenshot_manager.py | 50 --- dendrite/browser/sync_api/_core/_type_spec.py | 35 -- dendrite/browser/sync_api/_core/_utils.py | 93 ----- .../sync_api/_core/dendrite_browser.py | 395 ------------------ .../sync_api/_core/dendrite_element.py | 239 ----------- .../browser/sync_api/_core/dendrite_page.py | 377 ----------------- dendrite/browser/sync_api/_core/event_sync.py | 45 -- .../browser/sync_api/_core/mixin/__init__.py | 21 - dendrite/browser/sync_api/_core/mixin/ask.py | 189 --------- .../browser/sync_api/_core/mixin/click.py | 56 --- .../browser/sync_api/_core/mixin/extract.py | 284 ------------- .../sync_api/_core/mixin/fill_fields.py | 76 ---- .../sync_api/_core/mixin/get_element.py | 336 --------------- .../browser/sync_api/_core/mixin/keyboard.py | 62 --- .../browser/sync_api/_core/mixin/markdown.py | 23 - .../sync_api/_core/mixin/screenshot.py | 20 - .../browser/sync_api/_core/mixin/wait_for.py | 53 --- .../browser/sync_api/_core/models/__init__.py | 0 .../sync_api/_core/models/authentication.py | 47 --- .../_core/models/download_interface.py | 20 - .../_core/models/page_diff_information.py | 0 .../sync_api/_core/models/page_information.py | 15 - .../browser/sync_api/_core/models/response.py | 54 --- .../sync_api/_core/protocol/page_protocol.py | 19 - dendrite/browser/sync_api/_dom/__init__.py | 0 .../browser/sync_api/_remote_impl/__init__.py | 3 - .../_remote_impl/browserbase/__init__.py | 3 - .../_remote_impl/browserbase/_client.py | 63 --- .../_remote_impl/browserbase/_download.py | 53 --- .../_remote_impl/browserbase/_impl.py | 64 --- .../_remote_impl/browserless/__init__.py | 0 .../_remote_impl/browserless/_impl.py | 57 --- dendrite/logic/config.py | 7 +- 59 files changed, 290 insertions(+), 3406 deletions(-) delete mode 100644 dendrite/browser/async_api/_core/models/authentication.py delete mode 100644 dendrite/browser/async_api/_core/models/page_diff_information.py delete mode 100644 dendrite/browser/async_api/_core/models/page_information.py delete mode 100644 dendrite/browser/sync_api/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/_impl_browser.py delete mode 100644 dendrite/browser/sync_api/_core/_impl_mapping.py delete mode 100644 dendrite/browser/sync_api/_core/_js/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/_js/eventListenerPatch.js delete mode 100644 dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js delete mode 100644 dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js delete mode 100644 dendrite/browser/sync_api/_core/_local_browser_impl.py delete mode 100644 dendrite/browser/sync_api/_core/_managers/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/_managers/navigation_tracker.py delete mode 100644 dendrite/browser/sync_api/_core/_managers/page_manager.py delete mode 100644 dendrite/browser/sync_api/_core/_managers/screenshot_manager.py delete mode 100644 dendrite/browser/sync_api/_core/_type_spec.py delete mode 100644 dendrite/browser/sync_api/_core/_utils.py delete mode 100644 dendrite/browser/sync_api/_core/dendrite_browser.py delete mode 100644 dendrite/browser/sync_api/_core/dendrite_element.py delete mode 100644 dendrite/browser/sync_api/_core/dendrite_page.py delete mode 100644 dendrite/browser/sync_api/_core/event_sync.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/ask.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/click.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/extract.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/fill_fields.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/get_element.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/keyboard.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/markdown.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/screenshot.py delete mode 100644 dendrite/browser/sync_api/_core/mixin/wait_for.py delete mode 100644 dendrite/browser/sync_api/_core/models/__init__.py delete mode 100644 dendrite/browser/sync_api/_core/models/authentication.py delete mode 100644 dendrite/browser/sync_api/_core/models/download_interface.py delete mode 100644 dendrite/browser/sync_api/_core/models/page_diff_information.py delete mode 100644 dendrite/browser/sync_api/_core/models/page_information.py delete mode 100644 dendrite/browser/sync_api/_core/models/response.py delete mode 100644 dendrite/browser/sync_api/_core/protocol/page_protocol.py delete mode 100644 dendrite/browser/sync_api/_dom/__init__.py delete mode 100644 dendrite/browser/sync_api/_remote_impl/__init__.py delete mode 100644 dendrite/browser/sync_api/_remote_impl/browserbase/__init__.py delete mode 100644 dendrite/browser/sync_api/_remote_impl/browserbase/_client.py delete mode 100644 dendrite/browser/sync_api/_remote_impl/browserbase/_download.py delete mode 100644 dendrite/browser/sync_api/_remote_impl/browserbase/_impl.py delete mode 100644 dendrite/browser/sync_api/_remote_impl/browserless/__init__.py delete mode 100644 dendrite/browser/sync_api/_remote_impl/browserless/_impl.py diff --git a/dendrite/_cli/main.py b/dendrite/_cli/main.py index 370e4de..e2ec376 100644 --- a/dendrite/_cli/main.py +++ b/dendrite/_cli/main.py @@ -1,6 +1,9 @@ import argparse import subprocess import sys +import asyncio +from dendrite.browser.async_api import AsyncDendrite +from dendrite.logic.config import Config def run_playwright_install(): @@ -17,14 +20,36 @@ def run_playwright_install(): sys.exit(1) +async def setup_auth(url: str, profile_name: str): + try: + async with AsyncDendrite() as browser: + await browser.setup_auth( + url=url, + profile_name=profile_name, + message="Please log in to the website. Once done, press Enter to continue..." + ) + print(f"Authentication profile '{profile_name}' has been saved successfully.") + except Exception as e: + print(f"Error during authentication setup: {e}") + sys.exit(1) + + def main(): parser = argparse.ArgumentParser(description="Dendrite SDK CLI tool") - parser.add_argument("command", choices=["install"], help="Command to execute") + parser.add_argument("command", choices=["install", "auth"], help="Command to execute") + + # Add auth-specific arguments + parser.add_argument("--url", help="URL to navigate to for authentication") + parser.add_argument("--profile", default="default", help="Name for the authentication profile (default: 'default')") args = parser.parse_args() if args.command == "install": run_playwright_install() + elif args.command == "auth": + if not args.url: + parser.error("The --url argument is required for the auth command") + asyncio.run(setup_auth(args.url, args.profile)) if __name__ == "__main__": diff --git a/dendrite/browser/async_api/_core/_impl_browser.py b/dendrite/browser/async_api/_core/_impl_browser.py index 6754927..aafa93c 100644 --- a/dendrite/browser/async_api/_core/_impl_browser.py +++ b/dendrite/browser/async_api/_core/_impl_browser.py @@ -1,10 +1,11 @@ from abc import ABC, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional, Union, overload +from typing_extensions import Literal if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from playwright.async_api import Browser, Download, Playwright +from playwright.async_api import Browser, Download, Playwright, BrowserContext from dendrite.browser.async_api._core._type_spec import PlaywrightPage @@ -29,16 +30,35 @@ async def get_download( Exception: If there is an issue retrieving the download event. """ + @overload @abstractmethod - async def start_browser(self, playwright: Playwright, pw_options: dict) -> Browser: + async def start_browser( + self, playwright: Playwright, pw_options: dict, user_data_dir: str + ) -> BrowserContext: ... + + @overload + @abstractmethod + async def start_browser( + self, playwright: Playwright, pw_options: dict, user_data_dir: None = None + ) -> Browser: ... + + @abstractmethod + async def start_browser( + self, + playwright: Playwright, + pw_options: dict, + user_data_dir: Optional[str] = None, + ) -> Union[Browser, BrowserContext]: """ Starts the browser session. - Returns: - Browser: The browser session. + Args: + playwright: The playwright instance + pw_options: Playwright launch options + user_data_dir: Optional path to Chrome user data directory for persistent context - Raises: - Exception: If there is an issue starting the browser session. + Returns: + Union[Browser, BrowserContext]: Either a Browser instance or BrowserContext for persistent sessions """ @abstractmethod diff --git a/dendrite/browser/async_api/_core/_local_browser_impl.py b/dendrite/browser/async_api/_core/_local_browser_impl.py index a524ae6..aa5fd0a 100644 --- a/dendrite/browser/async_api/_core/_local_browser_impl.py +++ b/dendrite/browser/async_api/_core/_local_browser_impl.py @@ -1,19 +1,88 @@ -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union, overload +from loguru import logger +from typing_extensions import Literal + +from dendrite.browser._common.constants import STEALTH_ARGS if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from playwright.async_api import Browser, Download, Playwright +from playwright.async_api import Browser, Download, Playwright, BrowserContext from dendrite.browser.async_api._core._impl_browser import ImplBrowser from dendrite.browser.async_api._core._type_spec import PlaywrightPage +import tempfile +import shutil +import os + class LocalImpl(ImplBrowser): def __init__(self) -> None: pass - async def start_browser(self, playwright: Playwright, pw_options) -> Browser: + @overload + async def start_browser( + self, playwright: Playwright, pw_options: dict, user_data_dir: str + ) -> BrowserContext: ... + + @overload + async def start_browser( + self, playwright: Playwright, pw_options: dict, user_data_dir: None = None + ) -> Browser: ... + + async def start_browser( + self, + playwright: Playwright, + pw_options: dict, + user_data_dir: Optional[str] = None, + ) -> Union[Browser, BrowserContext]: + if user_data_dir is not None: + args = { + "user_data_dir": user_data_dir, + "ignore_default_args": ["--enable-automation"], + } + + # Check if profile is locked + singleton_lock = os.path.join(user_data_dir, "SingletonLock") + logger.warning(f"Checking for singleton lock at {os.path.abspath(singleton_lock)}") + + res = Path("/Users/arian/Projects/dendrite/dendrite-python-sdk/browser_profiles/my_google_profile/SingletonLock").is_symlink() + if res: + is_locked = True + else: + is_locked = False + + logger.warning(f"Profile is locked: {is_locked}") + if is_locked: + logger.warning("Profile is locked, creating a temporary copy") + # Create a temporary copy of the user data directory + temp_dir = tempfile.mkdtemp() + temp_user_data = os.path.join(temp_dir, "chrome_data") + + def copy_ignore(src, names): + return [ + 'SingletonSocket', 'SingletonLock', 'SingletonCookie', + 'DeferredBrowserMetrics', 'RunningChromeVersion', + '.org.chromium.Chromium.*' + ] + + # Copy tree with error handling + shutil.copytree( + user_data_dir, + temp_user_data, + ignore=copy_ignore, + dirs_exist_ok=True + ) + args["user_data_dir"] = temp_user_data + + pw_options.update(args) + return await playwright.chromium.launch_persistent_context( + channel="chrome", + **pw_options + ) + return await playwright.chromium.launch(**pw_options) async def get_download( diff --git a/dendrite/browser/async_api/_core/_managers/page_manager.py b/dendrite/browser/async_api/_core/_managers/page_manager.py index 34bf20b..25657f1 100644 --- a/dendrite/browser/async_api/_core/_managers/page_manager.py +++ b/dendrite/browser/async_api/_core/_managers/page_manager.py @@ -17,6 +17,17 @@ def __init__(self, dendrite_browser, browser_context: BrowserContext): self.browser_context = browser_context self.dendrite_browser: AsyncDendrite = dendrite_browser + # Handle existing pages in the context + existing_pages = browser_context.pages + if existing_pages: + for page in existing_pages: + client = self.dendrite_browser.logic_engine + dendrite_page = AsyncPage(page, self.dendrite_browser, client) + self.pages.append(dendrite_page) + # Set the first existing page as active + if self.active_page is None: + self.active_page = dendrite_page + browser_context.on("page", self._page_on_open_handler) async def new_page(self) -> AsyncPage: @@ -26,7 +37,7 @@ async def new_page(self) -> AsyncPage: if self.active_page and new_page == self.active_page.playwright_page: return self.active_page - client = self.dendrite_browser._get_logic_api() + client = self.dendrite_browser.logic_engine dendrite_page = AsyncPage(new_page, self.dendrite_browser, client) self.pages.append(dendrite_page) self.active_page = dendrite_page @@ -76,7 +87,7 @@ def _page_on_open_handler(self, page: PlaywrightPage): page.on("download", self._page_on_download_handler) page.on("filechooser", self._page_on_filechooser_handler) - client = self.dendrite_browser._get_logic_api() + client = self.dendrite_browser.logic_engine dendrite_page = AsyncPage(page, self.dendrite_browser, client) self.pages.append(dendrite_page) self.active_page = dendrite_page diff --git a/dendrite/browser/async_api/_core/_utils.py b/dendrite/browser/async_api/_core/_utils.py index 1db3237..e9074f1 100644 --- a/dendrite/browser/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_core/_utils.py @@ -16,6 +16,39 @@ from dendrite.browser.async_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT from dendrite.logic.dom.strip import mild_strip_in_place +import os +import platform +from pathlib import Path + + +def get_chrome_user_data_dir() -> str: + """ + Get the default Chrome user data directory based on the operating system. + + Returns: + str: Path to Chrome user data directory + """ + system = platform.system() + home = Path.home() + + if system == "Windows": + return str(Path(os.getenv("LOCALAPPDATA", "")) / "Google/Chrome/User Data") + elif system == "Darwin": # macOS + return str(home / "Library/Application Support/Google/Chrome/") + elif system == "Linux": + return str(home / ".config/google-chrome") + else: + raise NotImplementedError(f"Unsupported operating system: {system}") + + +def chrome_profile_exists() -> bool: + """Check if a Chrome profile exists in the default location.""" + try: + user_data_dir = get_chrome_user_data_dir() + return Path(user_data_dir).exists() + except: + return False + async def expand_iframes( page: PlaywrightPage, diff --git a/dendrite/browser/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/_core/dendrite_browser.py index 8f902cc..88837ae 100644 --- a/dendrite/browser/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/_core/dendrite_browser.py @@ -4,6 +4,9 @@ from abc import ABC from typing import Any, List, Optional, Sequence, Union from uuid import uuid4 +import shutil +import tempfile +import uuid from loguru import logger from playwright.async_api import ( @@ -42,6 +45,7 @@ from dendrite.browser.remote import Providers from dendrite.logic.config import Config from dendrite.logic.interfaces import AsyncProtocol +from ._utils import get_chrome_user_data_dir, chrome_profile_exists class AsyncDendrite( @@ -87,6 +91,7 @@ def __init__( }, remote_config: Optional[Providers] = None, config: Optional[Config] = None, + use_chrome_profile: bool = False, ): """ Initializes AsyncDendrite with API keys and Playwright options. @@ -96,6 +101,9 @@ def __init__( openai_api_key (Optional[str]): Your own OpenAI API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. anthropic_api_key (Optional[str]): The own Anthropic API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. playwright_options (Any): Options for configuring Playwright. Defaults to running in non-headless mode with stealth arguments. + remote_config (Optional[Providers]): Remote browser provider configuration + config (Optional[Config]): Configuration object + use_chrome_profile (bool): Whether to try using the existing Chrome profile. Defaults to False. Raises: MissingApiKeyError: If the Dendrite API key is not provided or found in the environment variables. @@ -122,6 +130,8 @@ def __init__( self.closed = False self._config = config or Config() self._browser_api_client: AsyncProtocol = AsyncProtocol(self._config) + self._use_chrome_profile = use_chrome_profile + self._temp_profile_dir = None @property def pages(self) -> List[AsyncPage]: @@ -140,10 +150,12 @@ async def _get_page(self) -> AsyncPage: active_page = await self.get_active_page() return active_page - def _get_logic_api(self) -> AsyncProtocol: + @property + def logic_engine(self) -> AsyncProtocol: return self._browser_api_client - def _get_dendrite_browser(self) -> "AsyncDendrite": + @property + def dendrite_browser(self) -> "AsyncDendrite": return self async def __aenter__(self): @@ -279,20 +291,27 @@ async def _launch(self): os.environ["PW_TEST_SCREENSHOT_NO_FONTS_READY"] = "1" self._playwright = await async_playwright().start() - # browser = await self._playwright.chromium.launch(**self._playwright_options) + browser = None + # Create temporary copy of Chrome profile if requested + if self._use_chrome_profile and chrome_profile_exists(): + original_dir = get_chrome_user_data_dir() - browser = await self._impl.start_browser( - self._playwright, self._playwright_options - ) + context_options = {**self._playwright_options} - self.browser_context = ( - browser.contexts[0] - if len(browser.contexts) > 0 - else await browser.new_context() - ) + self.browser_context = await self._impl.start_browser( + self._playwright, context_options, "browser_profiles/my_google_profile" + ) + else: + browser = await self._impl.start_browser( + self._playwright, self._playwright_options + ) + self.browser_context = ( + browser.contexts[0] + if len(browser.contexts) > 0 + else await browser.new_context() + ) self._active_page_manager = PageManager(self, self.browser_context) - await self._impl.configure_context(self) return browser, self.browser_context, self._active_page_manager @@ -314,7 +333,7 @@ async def add_cookies(self, cookies): async def close(self): """ - Closes the browser and uploads authentication session data if available. + Closes the browser and cleans up temporary profile directory if it exists. This method stops the Playwright instance, closes the browser context @@ -335,11 +354,16 @@ async def close(self): try: if self._playwright: await self._playwright.stop() - except AttributeError: - pass - except Exception: + except (AttributeError, Exception): pass + # Clean up temporary profile directory + if self._temp_profile_dir and os.path.exists(self._temp_profile_dir): + try: + shutil.rmtree(self._temp_profile_dir) + except Exception as e: + logger.warning(f"Failed to remove temporary profile directory: {e}") + def _is_launched(self): """ Checks whether the browser context has been launched. @@ -434,3 +458,60 @@ async def _get_filechooser( Exception: If there is an issue uploading files. """ return await self._upload_handler.get_data(pw_page, timeout=timeout) + + async def setup_auth( + self, + url: str, + message: str = "Please log in to the website. Once done, press Enter to continue...", + profile_name: str = "default", + ) -> None: + """ + Launches a browser session for user login and saves the profile data. + + Args: + profile_name (str): Name for the profile to be saved + url (str): URL to navigate to for login + message (str): Custom message to show while waiting for user input + + Returns: + None + """ + # Create profiles directory if it doesn't exist + profiles_dir = self._config.auth_session_path + profile_path = profiles_dir / profile_name + profiles_dir.mkdir(exist_ok=True) + + # Launch browser with temporary profile + + try: + # Start Playwright + self._playwright = await async_playwright().start() + + # Launch persistent context + context_options = { + "headless": False, # Always show browser for login + "args": STEALTH_ARGS, + } + + self.browser_context = ( + await self._playwright.chromium.launch_persistent_context( + user_data_dir=profile_path, + ignore_default_args=["--enable-automation"], + channel="chrome", + **context_options, + ) + ) + + # Set up page manager + self._active_page_manager = PageManager(self, self.browser_context) + + # Navigate to login page + await self.goto(url) + + # Wait for user to complete login + print(message) + input() + + finally: + # Clean up + await self.close() diff --git a/dendrite/browser/async_api/_core/dendrite_page.py b/dendrite/browser/async_api/_core/dendrite_page.py index 60bb570..b770dad 100644 --- a/dendrite/browser/async_api/_core/dendrite_page.py +++ b/dendrite/browser/async_api/_core/dendrite_page.py @@ -58,10 +58,10 @@ def __init__( ): self.playwright_page = page self.screenshot_manager = ScreenshotManager(page) - self.dendrite_browser = dendrite_browser self._browser_api_client = browser_api_client self._last_main_frame_url = page.url self._last_frame_navigated_timestamp = time.time() + self._dendrite_browser = dendrite_browser self.playwright_page.on("framenavigated", self._on_frame_navigated) @@ -70,6 +70,10 @@ def _on_frame_navigated(self, frame): self._last_main_frame_url = frame.url self._last_frame_navigated_timestamp = time.time() + @property + def dendrite_browser(self) -> "AsyncDendrite": + return self._dendrite_browser + @property def url(self): """ @@ -93,10 +97,8 @@ def keyboard(self) -> Keyboard: async def _get_page(self) -> "AsyncPage": return self - def _get_dendrite_browser(self) -> "AsyncDendrite": - return self.dendrite_browser - - def _get_logic_api(self) -> AsyncProtocol: + @property + def logic_engine(self) -> AsyncProtocol: return self._browser_api_client async def goto( diff --git a/dendrite/browser/async_api/_core/mixin/ask.py b/dendrite/browser/async_api/_core/mixin/ask.py index 5a4c217..8f86e5c 100644 --- a/dendrite/browser/async_api/_core/mixin/ask.py +++ b/dendrite/browser/async_api/_core/mixin/ask.py @@ -186,7 +186,7 @@ async def ask( ) try: - res = await self._get_logic_api().ask_page(dto) + res = await self.logic_engine.ask_page(dto) logger.debug(f"Got response in {time.time() - attempt_start} seconds") if res.status == "error": diff --git a/dendrite/browser/async_api/_core/mixin/extract.py b/dendrite/browser/async_api/_core/mixin/extract.py index cc3221b..093de34 100644 --- a/dendrite/browser/async_api/_core/mixin/extract.py +++ b/dendrite/browser/async_api/_core/mixin/extract.py @@ -173,7 +173,7 @@ async def _try_cached_extraction( """ page = await self._get_page() dto = CachedExtractDTO(url=page.url, prompt=prompt) - scripts = await self._get_logic_api().get_cached_scripts(dto) + scripts = await self.logic_engine.get_cached_scripts(dto) logger.debug(f"Found {len(scripts)} scripts in cache, {scripts}") if len(scripts) == 0: logger.debug( @@ -230,7 +230,7 @@ async def try_extract_with_agent(): use_screenshot=True, ) - res: ExtractResponse = await self._get_logic_api().extract(extract_dto) + res: ExtractResponse = await self.logic_engine.extract(extract_dto) if res.status == "impossible": logger.error(f"Impossible to extract data. Reason: {res.message}") diff --git a/dendrite/browser/async_api/_core/mixin/get_element.py b/dendrite/browser/async_api/_core/mixin/get_element.py index 5e9245a..a488f08 100644 --- a/dendrite/browser/async_api/_core/mixin/get_element.py +++ b/dendrite/browser/async_api/_core/mixin/get_element.py @@ -214,6 +214,7 @@ async def _get_element( if isinstance(prompt_or_elements, Dict): return None + logger.info(f"Getting element for prompt: '{prompt_or_elements}'") start_time = time.time() page = await self._get_page() soup = await page._get_soup() @@ -263,7 +264,7 @@ async def _try_cached_selectors( The found elements if successful, None otherwise """ dto = CachedSelectorDTO(url=page.url, prompt=prompt) - selectors = await self._get_logic_api().get_cached_selectors(dto) + selectors = await self.logic_engine.get_cached_selectors(dto) if len(selectors) == 0: logger.debug("No cached selectors found") @@ -345,7 +346,7 @@ async def _try_get_element(): prompt=prompt_or_elements, only_one=only_one, ) - res = await obj._get_logic_api().get_element(dto) + res = await obj.logic_engine.get_element(dto) if res.status == "impossible": logger.error( diff --git a/dendrite/browser/async_api/_core/models/authentication.py b/dendrite/browser/async_api/_core/models/authentication.py deleted file mode 100644 index 56992c9..0000000 --- a/dendrite/browser/async_api/_core/models/authentication.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import List, Literal, Optional - -from pydantic import BaseModel -from typing_extensions import TypedDict - - -class Cookie(TypedDict, total=False): - name: str - value: str - domain: str - path: str - expires: float - httpOnly: bool - secure: bool - sameSite: Literal["Lax", "None", "Strict"] - - -class LocalStorageEntry(TypedDict): - name: str - value: str - - -class OriginState(TypedDict): - origin: str - localStorage: List[LocalStorageEntry] - - -class StorageState(TypedDict, total=False): - cookies: List[Cookie] - origins: List[OriginState] - - -class DomainState(BaseModel): - domain: str - storage_state: StorageState - - -class AuthSession(BaseModel): - user_agent: Optional[str] - domain_states: List[DomainState] - - def to_storage_state(self) -> StorageState: - cookies = [] - origins = [] - for domain_state in self.domain_states: - cookies.extend(domain_state.storage_state.get("cookies", [])) - origins.extend(domain_state.storage_state.get("origins", [])) - return StorageState(cookies=cookies, origins=origins) diff --git a/dendrite/browser/async_api/_core/models/page_diff_information.py b/dendrite/browser/async_api/_core/models/page_diff_information.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/async_api/_core/models/page_information.py b/dendrite/browser/async_api/_core/models/page_information.py deleted file mode 100644 index 67d7892..0000000 --- a/dendrite/browser/async_api/_core/models/page_information.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import Optional - -from pydantic import BaseModel -from typing_extensions import TypedDict - - -class InteractableElementInfo(TypedDict): - attrs: Optional[str] - text: Optional[str] - - -class PageInformation(BaseModel): - url: str - raw_html: str - screenshot_base64: str - time_since_frame_navigated: float diff --git a/dendrite/browser/async_api/_core/protocol/page_protocol.py b/dendrite/browser/async_api/_core/protocol/page_protocol.py index b828af0..920434d 100644 --- a/dendrite/browser/async_api/_core/protocol/page_protocol.py +++ b/dendrite/browser/async_api/_core/protocol/page_protocol.py @@ -13,8 +13,10 @@ class DendritePageProtocol(Protocol): for the `ExtractionMixin` to work. """ - def _get_dendrite_browser(self) -> "AsyncDendrite": ... + @property + def logic_engine(self) -> AsyncProtocol: ... - def _get_logic_api(self) -> AsyncProtocol: ... + @property + def dendrite_browser(self) -> "AsyncDendrite": ... async def _get_page(self) -> "AsyncPage": ... diff --git a/dendrite/browser/sync_api/__init__.py b/dendrite/browser/sync_api/__init__.py deleted file mode 100644 index 3085e23..0000000 --- a/dendrite/browser/sync_api/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from loguru import logger -from ._core.dendrite_browser import Dendrite -from ._core.dendrite_element import Element -from ._core.dendrite_page import Page -from ._core.models.response import ElementsResponse - -__all__ = ["Dendrite", "Element", "Page", "ElementsResponse"] diff --git a/dendrite/browser/sync_api/_core/__init__.py b/dendrite/browser/sync_api/_core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_core/_impl_browser.py b/dendrite/browser/sync_api/_core/_impl_browser.py deleted file mode 100644 index b1d39a0..0000000 --- a/dendrite/browser/sync_api/_core/_impl_browser.py +++ /dev/null @@ -1,61 +0,0 @@ -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from playwright.sync_api import Browser, Download, Playwright -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage - - -class ImplBrowser(ABC): - - @abstractmethod - def __init__(self, settings): - pass - - @abstractmethod - def get_download( - self, dendrite_browser: "Dendrite", pw_page: PlaywrightPage, timeout: float - ) -> Download: - """ - Retrieves the download event from the browser. - - Returns: - Download: The download event. - - Raises: - Exception: If there is an issue retrieving the download event. - """ - - @abstractmethod - def start_browser(self, playwright: Playwright, pw_options: dict) -> Browser: - """ - Starts the browser session. - - Returns: - Browser: The browser session. - - Raises: - Exception: If there is an issue starting the browser session. - """ - - @abstractmethod - def configure_context(self, browser: "Dendrite") -> None: - """ - Configures the browser context. - - Args: - browser (Dendrite): The browser to configure. - - Raises: - Exception: If there is an issue configuring the browser context. - """ - - @abstractmethod - def stop_session(self) -> None: - """ - Stops the browser session. - - Raises: - Exception: If there is an issue stopping the browser session. - """ diff --git a/dendrite/browser/sync_api/_core/_impl_mapping.py b/dendrite/browser/sync_api/_core/_impl_mapping.py deleted file mode 100644 index 227b13e..0000000 --- a/dendrite/browser/sync_api/_core/_impl_mapping.py +++ /dev/null @@ -1,29 +0,0 @@ -from typing import Dict, Optional, Type -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser -from dendrite.browser.sync_api._core._local_browser_impl import LocalImpl -from dendrite.browser.sync_api._remote_impl.browserbase._impl import BrowserBaseImpl -from dendrite.browser.sync_api._remote_impl.browserless._impl import BrowserlessImpl -from dendrite.browser.remote import Providers -from dendrite.browser.remote.browserbase_config import BrowserbaseConfig -from dendrite.browser.remote.browserless_config import BrowserlessConfig - -IMPL_MAPPING: Dict[Type[Providers], Type[ImplBrowser]] = { - BrowserbaseConfig: BrowserBaseImpl, - BrowserlessConfig: BrowserlessImpl, -} -SETTINGS_CLASSES: Dict[str, Type[Providers]] = { - "browserbase": BrowserbaseConfig, - "browserless": BrowserlessConfig, -} - - -def get_impl(remote_provider: Optional[Providers]) -> ImplBrowser: - if remote_provider is None: - return LocalImpl() - try: - provider_class = IMPL_MAPPING[type(remote_provider)] - except KeyError: - raise ValueError( - f"No implementation for {type(remote_provider)}. Available providers: {', '.join(map(lambda x: x.__name__, IMPL_MAPPING.keys()))}" - ) - return provider_class(remote_provider) diff --git a/dendrite/browser/sync_api/_core/_js/__init__.py b/dendrite/browser/sync_api/_core/_js/__init__.py deleted file mode 100644 index ccaf080..0000000 --- a/dendrite/browser/sync_api/_core/_js/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from pathlib import Path - - -def load_script(filename: str) -> str: - current_dir = Path(__file__).parent - file_path = current_dir / filename - return file_path.read_text(encoding="utf-8") - - -GENERATE_DENDRITE_IDS_SCRIPT = load_script("generateDendriteIDs.js") -GENERATE_DENDRITE_IDS_IFRAME_SCRIPT = load_script("generateDendriteIDsIframe.js") diff --git a/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js b/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js deleted file mode 100644 index 7f03d55..0000000 --- a/dendrite/browser/sync_api/_core/_js/eventListenerPatch.js +++ /dev/null @@ -1,90 +0,0 @@ -// Save the original methods before redefining them -EventTarget.prototype._originalAddEventListener = EventTarget.prototype.addEventListener; -EventTarget.prototype._originalRemoveEventListener = EventTarget.prototype.removeEventListener; - -// Redefine the addEventListener method -EventTarget.prototype.addEventListener = function(event, listener, options = false) { - // Initialize the eventListenerList if it doesn't exist - if (!this.eventListenerList) { - this.eventListenerList = {}; - } - // Initialize the event list for the specific event if it doesn't exist - if (!this.eventListenerList[event]) { - this.eventListenerList[event] = []; - } - // Add the event listener details to the event list - this.eventListenerList[event].push({ listener, options, outerHTML: this.outerHTML }); - - // Call the original addEventListener method - this._originalAddEventListener(event, listener, options); -}; - -// Redefine the removeEventListener method -EventTarget.prototype.removeEventListener = function(event, listener, options = false) { - // Remove the event listener details from the event list - if (this.eventListenerList && this.eventListenerList[event]) { - this.eventListenerList[event] = this.eventListenerList[event].filter( - item => item.listener !== listener - ); - } - - // Call the original removeEventListener method - this._originalRemoveEventListener( event, listener, options); -}; - -// Get event listeners for a specific event type or all events if not specified -EventTarget.prototype._getEventListeners = function(eventType) { - if (!this.eventListenerList) { - this.eventListenerList = {}; - } - - const eventsToCheck = ['click', 'dblclick', 'mousedown', 'mouseup', 'mouseover', 'mouseout', 'mousemove', 'keydown', 'keyup', 'keypress']; - - eventsToCheck.forEach(type => { - if (!eventType || eventType === type) { - if (this[`on${type}`]) { - if (!this.eventListenerList[type]) { - this.eventListenerList[type] = []; - } - this.eventListenerList[type].push({ listener: this[`on${type}`], inline: true }); - } - } - }); - - return eventType === undefined ? this.eventListenerList : this.eventListenerList[eventType]; -}; - -// Utility to show events -function _showEvents(events) { - let result = ''; - for (let event in events) { - result += `${event} ----------------> ${events[event].length}\n`; - for (let listenerObj of events[event]) { - result += `${listenerObj.listener.toString()}\n`; - } - } - return result; -} - -// Extend EventTarget prototype with utility methods -EventTarget.prototype.on = function(event, callback, options) { - this.addEventListener(event, callback, options); - return this; -}; - -EventTarget.prototype.off = function(event, callback, options) { - this.removeEventListener(event, callback, options); - return this; -}; - -EventTarget.prototype.emit = function(event, args = null) { - this.dispatchEvent(new CustomEvent(event, { detail: args })); - return this; -}; - -// Make these methods non-enumerable -Object.defineProperties(EventTarget.prototype, { - on: { enumerable: false }, - off: { enumerable: false }, - emit: { enumerable: false } -}); diff --git a/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js b/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js deleted file mode 100644 index 0ae5f61..0000000 --- a/dendrite/browser/sync_api/_core/_js/generateDendriteIDs.js +++ /dev/null @@ -1,88 +0,0 @@ -var hashCode = (str) => { - var hash = 0, i, chr; - if (str.length === 0) return hash; - for (i = 0; i < str.length; i++) { - chr = str.charCodeAt(i); - hash = ((hash << 5) - hash) + chr; - hash |= 0; // Convert to 32bit integer - } - return hash; -} - - -const getElementIndex = (element) => { - let index = 1; - let sibling = element.previousElementSibling; - - while (sibling) { - if (sibling.localName === element.localName) { - index++; - } - sibling = sibling.previousElementSibling; - } - - return index; -}; - - -const segs = function elmSegs(elm) { - if (!elm || elm.nodeType !== 1) return ['']; - if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; - const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; - let index = getElementIndex(elm); - - return [...elmSegs(elm.parentNode), `${localName}[${index}]`]; -}; - -var getXPathForElement = (element) => { - return segs(element).join('/'); -} - -// Create a Map to store used hashes and their counters -const usedHashes = new Map(); - -var markHidden = (hidden_element) => { - // Mark the hidden element itself - hidden_element.setAttribute('data-hidden', 'true'); - -} - -document.querySelectorAll('*').forEach((element, index) => { - try { - - const xpath = getXPathForElement(element); - const hash = hashCode(xpath); - const baseId = hash.toString(36); - - // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; - const isHidden = !element.checkVisibility(); - // computedStyle.width === '0px' || - // computedStyle.height === '0px'; - - if (isHidden) { - markHidden(element); - }else{ - element.removeAttribute("data-hidden") // in case we hid it in a previous call - } - - let uniqueId = baseId; - let counter = 0; - - // Check if this hash has been used before - while (usedHashes.has(uniqueId)) { - // If it has, increment the counter and create a new uniqueId - counter++; - uniqueId = `${baseId}_${counter}`; - } - - // Add the uniqueId to the usedHashes Map - usedHashes.set(uniqueId, true); - element.setAttribute('d-id', uniqueId); - } catch (error) { - // Fallback: use a hash of the tag name and index - const fallbackId = hashCode(`${element.tagName}_${index}`).toString(36); - console.error('Error processing element, using fallback:',fallbackId, element, error); - - element.setAttribute('d-id', `fallback_${fallbackId}`); - } -}); \ No newline at end of file diff --git a/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js b/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js deleted file mode 100644 index 4f59cef..0000000 --- a/dendrite/browser/sync_api/_core/_js/generateDendriteIDsIframe.js +++ /dev/null @@ -1,93 +0,0 @@ -({frame_path}) => { - var hashCode = (str) => { - var hash = 0, i, chr; - if (str.length === 0) return hash; - for (i = 0; i < str.length; i++) { - chr = str.charCodeAt(i); - hash = ((hash << 5) - hash) + chr; - hash |= 0; // Convert to 32bit integer - } - return hash; - } - - const getElementIndex = (element) => { - let index = 1; - let sibling = element.previousElementSibling; - - while (sibling) { - if (sibling.localName === element.localName) { - index++; - } - sibling = sibling.previousElementSibling; - } - - return index; - }; - - - const segs = function elmSegs(elm) { - if (!elm || elm.nodeType !== 1) return ['']; - if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; - const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; - let index = getElementIndex(elm); - - return [...elmSegs(elm.parentNode), `${localName}[${index}]`]; - }; - - var getXPathForElement = (element) => { - return segs(element).join('/'); - } - - // Create a Map to store used hashes and their counters - const usedHashes = new Map(); - - var markHidden = (hidden_element) => { - // Mark the hidden element itself - hidden_element.setAttribute('data-hidden', 'true'); - } - - document.querySelectorAll('*').forEach((element, index) => { - try { - - - // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; - const isHidden = !element.checkVisibility(); - // computedStyle.width === '0px' || - // computedStyle.height === '0px'; - - if (isHidden) { - markHidden(element); - }else{ - element.removeAttribute("data-hidden") // in case we hid it in a previous call - } - let xpath = getXPathForElement(element); - if(frame_path){ - element.setAttribute("iframe-path",frame_path) - xpath = frame_path + xpath; - } - const hash = hashCode(xpath); - const baseId = hash.toString(36); - - let uniqueId = baseId; - let counter = 0; - - // Check if this hash has been used before - while (usedHashes.has(uniqueId)) { - // If it has, increment the counter and create a new uniqueId - counter++; - uniqueId = `${baseId}_${counter}`; - } - - // Add the uniqueId to the usedHashes Map - usedHashes.set(uniqueId, true); - element.setAttribute('d-id', uniqueId); - } catch (error) { - // Fallback: use a hash of the tag name and index - const fallbackId = hashCode(`${element.tagName}_${index}`).toString(36); - console.error('Error processing element, using fallback:',fallbackId, element, error); - - element.setAttribute('d-id', `fallback_${fallbackId}`); - } - }); -} - diff --git a/dendrite/browser/sync_api/_core/_local_browser_impl.py b/dendrite/browser/sync_api/_core/_local_browser_impl.py deleted file mode 100644 index 464a90e..0000000 --- a/dendrite/browser/sync_api/_core/_local_browser_impl.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from playwright.sync_api import Browser, Download, Playwright -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage - - -class LocalImpl(ImplBrowser): - - def __init__(self) -> None: - pass - - def start_browser(self, playwright: Playwright, pw_options) -> Browser: - return playwright.chromium.launch(**pw_options) - - def get_download( - self, dendrite_browser: "Dendrite", pw_page: PlaywrightPage, timeout: float - ) -> Download: - return dendrite_browser._download_handler.get_data(pw_page, timeout) - - def configure_context(self, browser: "Dendrite"): - pass - - def stop_session(self): - pass diff --git a/dendrite/browser/sync_api/_core/_managers/__init__.py b/dendrite/browser/sync_api/_core/_managers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py b/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py deleted file mode 100644 index 8dc632b..0000000 --- a/dendrite/browser/sync_api/_core/_managers/navigation_tracker.py +++ /dev/null @@ -1,67 +0,0 @@ -import time -import time -from typing import TYPE_CHECKING, Dict, Optional - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_page import Page - - -class NavigationTracker: - - def __init__(self, page: "Page"): - self.playwright_page = page.playwright_page - self._nav_start_timestamp: Optional[float] = None - self.playwright_page.on("framenavigated", self._on_frame_navigated) - self.playwright_page.on("popup", self._on_popup) - self._last_events: Dict[str, Optional[float]] = { - "framenavigated": None, - "popup": None, - } - - def _on_frame_navigated(self, frame): - self._last_events["framenavigated"] = time.time() - if frame is self.playwright_page.main_frame: - self._last_main_frame_url = frame.url - self._last_frame_navigated_timestamp = time.time() - - def _on_popup(self, page): - self._last_events["popup"] = time.time() - - def start_nav_tracking(self): - """Call this just before performing an action that might trigger navigation""" - self._nav_start_timestamp = time.time() - for event in self._last_events: - self._last_events[event] = None - - def get_nav_events_since_start(self): - """ - Returns which events have fired since start_nav_tracking() was called - and how long after the start they occurred - """ - if self._nav_start_timestamp is None: - return "Navigation tracking not started. Call start_nav_tracking() first." - results = {} - for event, timestamp in self._last_events.items(): - if timestamp is not None: - delay = timestamp - self._nav_start_timestamp - results[event] = f"{delay:.3f}s" - else: - results[event] = "not fired" - return results - - def has_navigated_since_start(self): - """Returns True if any navigation event has occurred since start_nav_tracking()""" - if self._nav_start_timestamp is None: - return False - start_time = time.time() - max_wait = 1.0 - while time.time() - start_time < max_wait: - if any( - ( - timestamp is not None and timestamp > self._nav_start_timestamp - for timestamp in self._last_events.values() - ) - ): - return True - time.sleep(0.1) - return False diff --git a/dendrite/browser/sync_api/_core/_managers/page_manager.py b/dendrite/browser/sync_api/_core/_managers/page_manager.py deleted file mode 100644 index ad25fa6..0000000 --- a/dendrite/browser/sync_api/_core/_managers/page_manager.py +++ /dev/null @@ -1,74 +0,0 @@ -from typing import TYPE_CHECKING, Optional -from loguru import logger -from playwright.sync_api import BrowserContext, Download, FileChooser - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.sync_api._core.dendrite_page import Page - - -class PageManager: - - def __init__(self, dendrite_browser, browser_context: BrowserContext): - self.pages: list[Page] = [] - self.active_page: Optional[Page] = None - self.browser_context = browser_context - self.dendrite_browser: Dendrite = dendrite_browser - browser_context.on("page", self._page_on_open_handler) - - def new_page(self) -> Page: - new_page = self.browser_context.new_page() - if self.active_page and new_page == self.active_page.playwright_page: - return self.active_page - client = self.dendrite_browser._get_logic_api() - dendrite_page = Page(new_page, self.dendrite_browser, client) - self.pages.append(dendrite_page) - self.active_page = dendrite_page - return dendrite_page - - def get_active_page(self) -> Page: - if self.active_page is None: - return self.new_page() - return self.active_page - - def _page_on_close_handler(self, page: PlaywrightPage): - if self.browser_context and (not self.dendrite_browser.closed): - copy_pages = self.pages.copy() - is_active_page = False - for dendrite_page in copy_pages: - if dendrite_page.playwright_page == page: - self.pages.remove(dendrite_page) - if dendrite_page == self.active_page: - is_active_page = True - break - for i in reversed(range(len(self.pages))): - try: - self.active_page = self.pages[i] - self.pages[i].playwright_page.bring_to_front() - break - except Exception as e: - logger.warning(f"Error switching to the next page: {e}") - continue - - def _page_on_crash_handler(self, page: PlaywrightPage): - logger.error(f"Page crashed: {page.url}") - page.reload() - - def _page_on_download_handler(self, download: Download): - logger.debug(f"Download started: {download.url}") - self.dendrite_browser._download_handler.set_event(download) - - def _page_on_filechooser_handler(self, file_chooser: FileChooser): - logger.debug("File chooser opened") - self.dendrite_browser._upload_handler.set_event(file_chooser) - - def _page_on_open_handler(self, page: PlaywrightPage): - page.on("close", self._page_on_close_handler) - page.on("crash", self._page_on_crash_handler) - page.on("download", self._page_on_download_handler) - page.on("filechooser", self._page_on_filechooser_handler) - client = self.dendrite_browser._get_logic_api() - dendrite_page = Page(page, self.dendrite_browser, client) - self.pages.append(dendrite_page) - self.active_page = dendrite_page diff --git a/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py b/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py deleted file mode 100644 index 9a01f7c..0000000 --- a/dendrite/browser/sync_api/_core/_managers/screenshot_manager.py +++ /dev/null @@ -1,50 +0,0 @@ -import base64 -import os -from uuid import uuid4 -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage - - -class ScreenshotManager: - - def __init__(self, page: PlaywrightPage) -> None: - self.screenshot_before: str = "" - self.screenshot_after: str = "" - self.page = page - - def take_full_page_screenshot(self) -> str: - try: - scroll_height = self.page.evaluate( - "\n () => {\n const body = document.body;\n if (!body) {\n return 0; // Return 0 if body is null\n }\n return body.scrollHeight || 0;\n }\n " - ) - if scroll_height > 30000: - print( - f"Page height ({scroll_height}px) exceeds 30000px. Taking viewport screenshot instead." - ) - return self.take_viewport_screenshot() - image_data = self.page.screenshot( - type="jpeg", full_page=True, timeout=10000 - ) - except Exception as e: - print( - f"Full-page screenshot failed: {e}. Falling back to viewport screenshot." - ) - return self.take_viewport_screenshot() - if image_data is None: - return "" - return base64.b64encode(image_data).decode("utf-8") - - def take_viewport_screenshot(self) -> str: - image_data = self.page.screenshot(type="jpeg", timeout=10000) - if image_data is None: - return "" - reduced_base64 = base64.b64encode(image_data).decode("utf-8") - return reduced_base64 - - def store_screenshot(self, name, image_data): - if not name: - name = str(uuid4()) - filepath = os.path.join("test", f"{name}.jpeg") - os.makedirs(os.path.dirname(filepath), exist_ok=True) - with open(filepath, "wb") as file: - file.write(image_data) - return filepath diff --git a/dendrite/browser/sync_api/_core/_type_spec.py b/dendrite/browser/sync_api/_core/_type_spec.py deleted file mode 100644 index 7584120..0000000 --- a/dendrite/browser/sync_api/_core/_type_spec.py +++ /dev/null @@ -1,35 +0,0 @@ -import inspect -from typing import Any, Dict, Literal, Type, TypeVar, Union -from playwright.sync_api import Page -from pydantic import BaseModel - -Interaction = Literal["click", "fill", "hover"] -T = TypeVar("T") -PydanticModel = TypeVar("PydanticModel", bound=BaseModel) -PrimitiveTypes = PrimitiveTypes = Union[Type[bool], Type[int], Type[float], Type[str]] -JsonSchema = Dict[str, Any] -TypeSpec = Union[PrimitiveTypes, PydanticModel, JsonSchema] -PlaywrightPage = Page - - -def to_json_schema(type_spec: TypeSpec) -> Dict[str, Any]: - if isinstance(type_spec, dict): - return type_spec - if inspect.isclass(type_spec) and issubclass(type_spec, BaseModel): - return type_spec.model_json_schema() - if type_spec in (bool, int, float, str): - type_map = {bool: "boolean", int: "integer", float: "number", str: "string"} - return {"type": type_map[type_spec]} - raise ValueError(f"Unsupported type specification: {type_spec}") - - -def convert_to_type_spec(type_spec: TypeSpec, return_data: Any) -> TypeSpec: - if isinstance(type_spec, type): - if issubclass(type_spec, BaseModel): - return type_spec.model_validate(return_data) - if type_spec in (str, float, bool, int): - return type_spec(return_data) - raise ValueError(f"Unsupported type: {type_spec}") - if isinstance(type_spec, dict): - return return_data - raise ValueError(f"Unsupported type specification: {type_spec}") diff --git a/dendrite/browser/sync_api/_core/_utils.py b/dendrite/browser/sync_api/_core/_utils.py deleted file mode 100644 index 846ee40..0000000 --- a/dendrite/browser/sync_api/_core/_utils.py +++ /dev/null @@ -1,93 +0,0 @@ -from typing import TYPE_CHECKING, List, Optional, Union -from bs4 import BeautifulSoup -from loguru import logger -from playwright.sync_api import ElementHandle, Error, Frame, FrameLocator -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.sync_api._core.dendrite_element import Element -from dendrite.browser.sync_api._core.models.response import ElementsResponse -from dendrite.models.response.get_element_response import GetElementResponse -from dendrite.models.selector import Selector - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_page import Page -from dendrite.browser.sync_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT -from dendrite.logic.dom.strip import mild_strip_in_place - - -def expand_iframes(page: PlaywrightPage, page_soup: BeautifulSoup): - - def get_iframe_path(frame: Frame): - path_parts = [] - current_frame = frame - while current_frame.parent_frame is not None: - iframe_element = current_frame.frame_element() - iframe_id = iframe_element.get_attribute("d-id") - if iframe_id is None: - return None - path_parts.insert(0, iframe_id) - current_frame = current_frame.parent_frame - return "|".join(path_parts) - - for frame in page.frames: - if frame.parent_frame is None: - continue - try: - iframe_element = frame.frame_element() - iframe_id = iframe_element.get_attribute("d-id") - if iframe_id is None: - continue - iframe_path = get_iframe_path(frame) - except Error as e: - continue - if iframe_path is None: - continue - try: - frame.evaluate( - GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, {"frame_path": iframe_path} - ) - frame_content = frame.content() - frame_tree = BeautifulSoup(frame_content, "lxml") - mild_strip_in_place(frame_tree) - merge_iframe_to_page(iframe_id, page_soup, frame_tree) - except Error as e: - logger.debug(f"Error processing frame {iframe_id}: {e}") - continue - - -def merge_iframe_to_page(iframe_id: str, page: BeautifulSoup, iframe: BeautifulSoup): - iframe_element = page.find("iframe", {"d-id": iframe_id}) - if iframe_element is None: - logger.debug(f"Could not find iframe with ID {iframe_id} in page soup") - return - iframe_element.replace_with(iframe) - - -def _get_all_elements_from_selector_soup( - selector: str, soup: BeautifulSoup, page: "Page" -) -> List[Element]: - dendrite_elements: List[Element] = [] - elements = soup.select(selector) - for element in elements: - frame = page._get_context(element) - d_id = element.get("d-id", "") - locator = frame.locator(f"xpath=//*[@d-id='{d_id}']") - if not d_id: - continue - if isinstance(d_id, list): - d_id = d_id[0] - dendrite_elements.append( - Element(d_id, locator, page.dendrite_browser, page._browser_api_client) - ) - return dendrite_elements - - -def get_elements_from_selectors_soup( - page: "Page", soup: BeautifulSoup, selectors: List[Selector], only_one: bool -) -> Union[Optional[Element], List[Element]]: - for selector in reversed(selectors): - dendrite_elements = _get_all_elements_from_selector_soup( - selector.selector, soup, page - ) - if len(dendrite_elements) > 0: - return dendrite_elements[0] if only_one else dendrite_elements - return None diff --git a/dendrite/browser/sync_api/_core/dendrite_browser.py b/dendrite/browser/sync_api/_core/dendrite_browser.py deleted file mode 100644 index b37e35c..0000000 --- a/dendrite/browser/sync_api/_core/dendrite_browser.py +++ /dev/null @@ -1,395 +0,0 @@ -import os -import pathlib -import re -from abc import ABC -from typing import Any, List, Optional, Sequence, Union -from uuid import uuid4 -from loguru import logger -from playwright.sync_api import ( - BrowserContext, - Download, - Error, - FileChooser, - FilePayload, - Playwright, - sync_playwright, -) -from dendrite.browser._common._exceptions.dendrite_exception import ( - BrowserNotLaunchedError, - DendriteException, - IncorrectOutcomeError, -) -from dendrite.browser._common.constants import STEALTH_ARGS -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser -from dendrite.browser.sync_api._core._impl_mapping import get_impl -from dendrite.browser.sync_api._core._managers.page_manager import PageManager -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.sync_api._core.dendrite_page import Page -from dendrite.browser.sync_api._core.event_sync import EventSync -from dendrite.browser.sync_api._core.mixin import ( - AskMixin, - ClickMixin, - ExtractionMixin, - FillFieldsMixin, - GetElementMixin, - KeyboardMixin, - MarkdownMixin, - ScreenshotMixin, - WaitForMixin, -) -from dendrite.browser.remote import Providers -from dendrite.logic.config import Config -from dendrite.logic.interfaces import SyncProtocol - - -class Dendrite( - ScreenshotMixin, - WaitForMixin, - MarkdownMixin, - ExtractionMixin, - AskMixin, - FillFieldsMixin, - ClickMixin, - KeyboardMixin, - GetElementMixin, - ABC, -): - """ - Dendrite is a class that manages a browser instance using Playwright, allowing - interactions with web pages using natural language. - - This class handles initialization with API keys for Dendrite, OpenAI, and Anthropic, manages browser - contexts, and provides methods for navigation, authentication, and other browser-related tasks. - - Attributes: - id (UUID): The unique identifier for the Dendrite instance. - auth_data (Optional[AuthSession]): The authentication session data for the browser. - dendrite_api_key (str): The API key for Dendrite, used for interactions with the Dendrite API. - playwright_options (dict): Options for configuring the Playwright browser instance. - playwright (Optional[Playwright]): The Playwright instance managing the browser. - browser_context (Optional[BrowserContext]): The current browser context, which may include cookies and other session data. - active_page_manager (Optional[PageManager]): The manager responsible for handling active pages within the browser context. - user_id (Optional[str]): The user ID associated with the browser session. - browser_api_client (BrowserAPIClient): The API client used for communicating with the Dendrite API. - api_config (APIConfig): The configuration for the language models, including API keys for OpenAI and Anthropic. - - Raises: - Exception: If any of the required API keys (Dendrite, OpenAI, Anthropic) are not provided or found in the environment variables. - """ - - def __init__( - self, - playwright_options: Any = {"headless": False, "args": STEALTH_ARGS}, - remote_config: Optional[Providers] = None, - config: Optional[Config] = None, - ): - """ - Initializes Dendrite with API keys and Playwright options. - - Args: - dendrite_api_key (Optional[str]): The Dendrite API key. If not provided, it's fetched from the environment variables. - openai_api_key (Optional[str]): Your own OpenAI API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. - anthropic_api_key (Optional[str]): The own Anthropic API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. - playwright_options (Any): Options for configuring Playwright. Defaults to running in non-headless mode with stealth arguments. - - Raises: - MissingApiKeyError: If the Dendrite API key is not provided or found in the environment variables. - """ - self._impl = self._get_impl(remote_config) - self.playwright: Optional[Playwright] = None - self.browser_context: Optional[BrowserContext] = None - self._id = uuid4().hex - self._playwright_options = playwright_options - self._active_page_manager: Optional[PageManager] = None - self._user_id: Optional[str] = None - self._upload_handler = EventSync(event_type=FileChooser) - self._download_handler = EventSync(event_type=Download) - self.closed = False - self._config = config or Config() - self._browser_api_client: SyncProtocol = SyncProtocol(self._config) - - @property - def pages(self) -> List[Page]: - """ - Retrieves the list of active pages managed by the PageManager. - - Returns: - List[Page]: The list of active pages. - """ - if self._active_page_manager: - return self._active_page_manager.pages - else: - raise BrowserNotLaunchedError() - - def _get_page(self) -> Page: - active_page = self.get_active_page() - return active_page - - def _get_logic_api(self) -> SyncProtocol: - return self._browser_api_client - - def _get_dendrite_browser(self) -> "Dendrite": - return self - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def _get_impl(self, remote_provider: Optional[Providers]) -> ImplBrowser: - return get_impl(remote_provider) - - def get_active_page(self) -> Page: - """ - Retrieves the currently active page managed by the PageManager. - - Returns: - Page: The active page object. - - Raises: - Exception: If there is an issue retrieving the active page. - """ - active_page_manager = self._get_active_page_manager() - return active_page_manager.get_active_page() - - def new_tab( - self, url: str, timeout: Optional[float] = 15000, expected_page: str = "" - ) -> Page: - """ - Opens a new tab and navigates to the specified URL. - - Args: - url (str): The URL to navigate to. - timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. - expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. - - Returns: - Page: The page object after navigation. - - Raises: - Exception: If there is an error during navigation or if the expected page type is not found. - """ - return self.goto( - url, new_tab=True, timeout=timeout, expected_page=expected_page - ) - - def goto( - self, - url: str, - new_tab: bool = False, - timeout: Optional[float] = 15000, - expected_page: str = "", - ) -> Page: - """ - Navigates to the specified URL, optionally in a new tab - - Args: - url (str): The URL to navigate to. - new_tab (bool, optional): Whether to open the URL in a new tab. Defaults to False. - timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. - expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. - - Returns: - Page: The page object after navigation. - - Raises: - Exception: If there is an error during navigation or if the expected page type is not found. - """ - if not re.match("^\\w+://", url): - url = f"https://{url}" - active_page_manager = self._get_active_page_manager() - if new_tab: - active_page = active_page_manager.new_page() - else: - active_page = active_page_manager.get_active_page() - try: - logger.info(f"Going to {url}") - active_page.playwright_page.goto(url, timeout=timeout) - except TimeoutError: - logger.debug("Timeout when loading page but continuing anyways.") - except Exception as e: - logger.debug(f"Exception when loading page but continuing anyways. {e}") - if expected_page != "": - try: - prompt = f"We are checking if we have arrived on the expected type of page. If it is apparent that we have arrived on the wrong page, output an error. Here is the description: '{expected_page}'" - active_page.ask(prompt, bool) - except DendriteException as e: - raise IncorrectOutcomeError(f"Incorrect navigation, reason: {e}") - return active_page - - def scroll_to_bottom( - self, - timeout: float = 30000, - scroll_increment: int = 1000, - no_progress_limit: int = 3, - ): - """ - Scrolls to the bottom of the current page. - - Returns: - None - """ - active_page = self.get_active_page() - active_page.scroll_to_bottom( - timeout=timeout, - scroll_increment=scroll_increment, - no_progress_limit=no_progress_limit, - ) - - def _launch(self): - """ - Launches the Playwright instance and sets up the browser context and page manager. - - This method initializes the Playwright instance, creates a browser context, and sets up the PageManager. - It also applies any authentication data if available. - - Returns: - Tuple[Browser, BrowserContext, PageManager]: The launched browser, context, and page manager. - - Raises: - Exception: If there is an issue launching the browser or setting up the context. - """ - os.environ["PW_TEST_SCREENSHOT_NO_FONTS_READY"] = "1" - self._playwright = sync_playwright().start() - browser = self._impl.start_browser(self._playwright, self._playwright_options) - self.browser_context = ( - browser.contexts[0] if len(browser.contexts) > 0 else browser.new_context() - ) - self._active_page_manager = PageManager(self, self.browser_context) - self._impl.configure_context(self) - return (browser, self.browser_context, self._active_page_manager) - - def add_cookies(self, cookies): - """ - Adds cookies to the current browser context. - - Args: - cookies (List[Dict[str, Any]]): A list of cookies to be added to the browser context. - - Raises: - Exception: If the browser context is not initialized. - """ - if not self.browser_context: - raise DendriteException("Browser context not initialized") - self.browser_context.add_cookies(cookies) - - def close(self): - """ - Closes the browser and uploads authentication session data if available. - - This method stops the Playwright instance, closes the browser context - - Returns: - None - - Raises: - Exception: If there is an issue closing the browser or uploading session data. - """ - self.closed = True - try: - if self.browser_context: - self._impl.stop_session() - self.browser_context.close() - except Error: - pass - try: - if self._playwright: - self._playwright.stop() - except AttributeError: - pass - except Exception: - pass - - def _is_launched(self): - """ - Checks whether the browser context has been launched. - - Returns: - bool: True if the browser context is launched, False otherwise. - """ - return self.browser_context is not None - - def _get_active_page_manager(self) -> PageManager: - """ - Retrieves the active PageManager instance, launching the browser if necessary. - - Returns: - PageManager: The active PageManager instance. - - Raises: - Exception: If there is an issue launching the browser or retrieving the PageManager. - """ - if not self._active_page_manager: - (_, _, active_page_manager) = self._launch() - return active_page_manager - return self._active_page_manager - - def get_download(self, timeout: float) -> Download: - """ - Retrieves the download event from the browser. - - Returns: - Download: The download event. - - Raises: - Exception: If there is an issue retrieving the download event. - """ - active_page = self.get_active_page() - pw_page = active_page.playwright_page - return self._get_download(pw_page, timeout) - - def _get_download(self, pw_page: PlaywrightPage, timeout: float) -> Download: - """ - Retrieves the download event from the browser. - - Returns: - Download: The download event. - - Raises: - Exception: If there is an issue retrieving the download event. - """ - return self._download_handler.get_data(pw_page, timeout=timeout) - - def upload_files( - self, - files: Union[ - str, - pathlib.Path, - FilePayload, - Sequence[Union[str, pathlib.Path]], - Sequence[FilePayload], - ], - timeout: float = 30000, - ) -> None: - """ - Uploads files to the active page using a file chooser. - - Args: - files (Union[str, pathlib.Path, FilePayload, Sequence[Union[str, pathlib.Path]], Sequence[FilePayload]]): The file(s) to be uploaded. - This can be a file path, a `FilePayload` object, or a sequence of file paths or `FilePayload` objects. - timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the file chooser to be ready. Defaults to 30. - - Returns: - None - """ - page = self.get_active_page() - file_chooser = self._get_filechooser(page.playwright_page, timeout) - file_chooser.set_files(files) - - def _get_filechooser( - self, pw_page: PlaywrightPage, timeout: float = 30000 - ) -> FileChooser: - """ - Uploads files to the browser. - - Args: - timeout (float): The maximum time to wait for the file chooser dialog. Defaults to 30000 milliseconds. - - Returns: - FileChooser: The file chooser dialog. - - Raises: - Exception: If there is an issue uploading files. - """ - return self._upload_handler.get_data(pw_page, timeout=timeout) diff --git a/dendrite/browser/sync_api/_core/dendrite_element.py b/dendrite/browser/sync_api/_core/dendrite_element.py deleted file mode 100644 index cf9cfe5..0000000 --- a/dendrite/browser/sync_api/_core/dendrite_element.py +++ /dev/null @@ -1,239 +0,0 @@ -from __future__ import annotations -import time -import base64 -import functools -import time -from typing import TYPE_CHECKING, Optional -from loguru import logger -from playwright.sync_api import Locator -from dendrite.browser._common._exceptions.dendrite_exception import ( - IncorrectOutcomeError, -) -from dendrite.logic.interfaces import SyncProtocol - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser.sync_api._core._managers.navigation_tracker import ( - NavigationTracker, -) -from dendrite.browser.sync_api._core._type_spec import Interaction -from dendrite.models.dto.make_interaction_dto import VerifyActionDTO -from dendrite.models.response.interaction_response import InteractionResponse - - -def perform_action(interaction_type: Interaction): - """ - Decorator for performing actions on DendriteElements. - - This decorator wraps methods of Element to handle interactions, - expected outcomes, and error handling. - - Args: - interaction_type (Interaction): The type of interaction being performed. - - Returns: - function: The decorated function. - """ - - def decorator(func): - - @functools.wraps(func) - def wrapper(self: Element, *args, **kwargs) -> InteractionResponse: - expected_outcome: Optional[str] = kwargs.pop("expected_outcome", None) - if not expected_outcome: - func(self, *args, **kwargs) - return InteractionResponse(status="success", message="") - page_before = self._dendrite_browser.get_active_page() - page_before_info = page_before.get_page_information() - soup = page_before._get_previous_soup() - screenshot_before = page_before_info.screenshot_base64 - tag_name = soup.find(attrs={"d-id": self.dendrite_id}) - func(self, *args, expected_outcome=expected_outcome, **kwargs) - self._wait_for_page_changes(page_before.url) - page_after = self._dendrite_browser.get_active_page() - screenshot_after = page_after.screenshot_manager.take_full_page_screenshot() - dto = VerifyActionDTO( - url=page_before.url, - dendrite_id=self.dendrite_id, - interaction_type=interaction_type, - expected_outcome=expected_outcome, - screenshot_before=screenshot_before, - screenshot_after=screenshot_after, - tag_name=str(tag_name), - ) - res = self._browser_api_client.verify_action(dto) - if res.status == "failed": - raise IncorrectOutcomeError( - message=res.message, screenshot_base64=screenshot_after - ) - return res - - return wrapper - - return decorator - - -class Element: - """ - Represents an element in the Dendrite browser environment. Wraps a Playwright Locator. - - This class provides methods for interacting with and manipulating - elements in the browser. - """ - - def __init__( - self, - dendrite_id: str, - locator: Locator, - dendrite_browser: Dendrite, - browser_api_client: SyncProtocol, - ): - """ - Initialize a Element. - - Args: - dendrite_id (str): The dendrite_id identifier for this element. - locator (Locator): The Playwright locator for this element. - dendrite_browser (Dendrite): The browser instance. - """ - self.dendrite_id = dendrite_id - self.locator = locator - self._dendrite_browser = dendrite_browser - self._browser_api_client = browser_api_client - - def outer_html(self): - return self.locator.evaluate("(element) => element.outerHTML") - - def screenshot(self) -> str: - """ - Take a screenshot of the element and return it as a base64-encoded string. - - Returns: - str: A base64-encoded string of the JPEG image. - Returns an empty string if the screenshot fails. - """ - image_data = self.locator.screenshot(type="jpeg", timeout=20000) - if image_data is None: - return "" - return base64.b64encode(image_data).decode() - - @perform_action("click") - def click( - self, - expected_outcome: Optional[str] = None, - wait_for_navigation: bool = True, - *args, - **kwargs, - ) -> InteractionResponse: - """ - Click the element. - - Args: - expected_outcome (Optional[str]): The expected outcome of the click action. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - - Returns: - InteractionResponse: The response from the interaction. - """ - timeout = kwargs.pop("timeout", 2000) - force = kwargs.pop("force", False) - page = self._dendrite_browser.get_active_page() - navigation_tracker = NavigationTracker(page) - navigation_tracker.start_nav_tracking() - try: - self.locator.click(*args, timeout=timeout, force=force, **kwargs) - except Exception as e: - try: - self.locator.click(*args, timeout=2000, force=True, **kwargs) - except Exception as e: - self.locator.dispatch_event("click", timeout=2000) - if wait_for_navigation: - has_navigated = navigation_tracker.has_navigated_since_start() - if has_navigated: - try: - start_time = time.time() - page.playwright_page.wait_for_load_state("load", timeout=2000) - wait_duration = time.time() - start_time - except Exception as e: - pass - return InteractionResponse(status="success", message="") - - @perform_action("fill") - def fill( - self, value: str, expected_outcome: Optional[str] = None, *args, **kwargs - ) -> InteractionResponse: - """ - Fill the element with a value. If the element itself is not fillable, - it attempts to find and fill a fillable child element. - - Args: - value (str): The value to fill the element with. - expected_outcome (Optional[str]): The expected outcome of the fill action. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - - Returns: - InteractionResponse: The response from the interaction. - """ - timeout = kwargs.pop("timeout", 2000) - try: - self.locator.fill(value, *args, timeout=timeout, **kwargs) - except Exception as e: - fillable_child = self.locator.locator( - 'input, textarea, [contenteditable="true"]' - ).first - fillable_child.fill(value, *args, timeout=timeout, **kwargs) - return InteractionResponse(status="success", message="") - - @perform_action("hover") - def hover( - self, expected_outcome: Optional[str] = None, *args, **kwargs - ) -> InteractionResponse: - """ - Hover over the element. - All additional arguments are passed to the Playwright fill method. - - Args: - expected_outcome (Optional[str]): The expected outcome of the hover action. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - - Returns: - InteractionResponse: The response from the interaction. - """ - timeout = kwargs.pop("timeout", 2000) - self.locator.hover(*args, timeout=timeout, **kwargs) - return InteractionResponse(status="success", message="") - - def focus(self): - """ - Focus on the element. - """ - self.locator.focus() - - def highlight(self): - """ - Highlights the element. This is a convenience method for debugging purposes. - """ - self.locator.highlight() - - def _wait_for_page_changes(self, old_url: str, timeout: float = 2000): - """ - Wait for page changes after an action. - - Args: - old_url (str): The URL before the action. - timeout (float): The maximum time (in milliseconds) to wait for changes. - - Returns: - bool: True if the page changed, False otherwise. - """ - timeout_in_seconds = timeout / 1000 - start_time = time.time() - while time.time() - start_time <= timeout_in_seconds: - page = self._dendrite_browser.get_active_page() - if page.url != old_url: - return True - time.sleep(0.1) - return False diff --git a/dendrite/browser/sync_api/_core/dendrite_page.py b/dendrite/browser/sync_api/_core/dendrite_page.py deleted file mode 100644 index a1d7578..0000000 --- a/dendrite/browser/sync_api/_core/dendrite_page.py +++ /dev/null @@ -1,377 +0,0 @@ -import time -import pathlib -import re -import time -from typing import TYPE_CHECKING, Any, List, Literal, Optional, Sequence, Union -from bs4 import BeautifulSoup, Tag -from loguru import logger -from playwright.sync_api import Download, FilePayload, FrameLocator, Keyboard -from dendrite.browser.sync_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.sync_api._core.dendrite_element import Element -from dendrite.browser.sync_api._core.mixin.ask import AskMixin -from dendrite.browser.sync_api._core.mixin.click import ClickMixin -from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin -from dendrite.browser.sync_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.sync_api._core.mixin.keyboard import KeyboardMixin -from dendrite.browser.sync_api._core.mixin.markdown import MarkdownMixin -from dendrite.browser.sync_api._core.mixin.wait_for import WaitForMixin -from dendrite.logic.interfaces import SyncProtocol -from dendrite.models.page_information import PageInformation - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.sync_api._core._managers.screenshot_manager import ( - ScreenshotManager, -) -from dendrite.browser.sync_api._core._utils import expand_iframes - - -class Page( - MarkdownMixin, - ExtractionMixin, - WaitForMixin, - AskMixin, - FillFieldsMixin, - ClickMixin, - KeyboardMixin, - GetElementMixin, -): - """ - Represents a page in the Dendrite browser environment. - - This class provides methods for interacting with and manipulating - pages in the browser. - """ - - def __init__( - self, - page: PlaywrightPage, - dendrite_browser: "Dendrite", - browser_api_client: SyncProtocol, - ): - self.playwright_page = page - self.screenshot_manager = ScreenshotManager(page) - self.dendrite_browser = dendrite_browser - self._browser_api_client = browser_api_client - self._last_main_frame_url = page.url - self._last_frame_navigated_timestamp = time.time() - self.playwright_page.on("framenavigated", self._on_frame_navigated) - - def _on_frame_navigated(self, frame): - if frame is self.playwright_page.main_frame: - self._last_main_frame_url = frame.url - self._last_frame_navigated_timestamp = time.time() - - @property - def url(self): - """ - Get the current URL of the page. - - Returns: - str: The current URL. - """ - return self.playwright_page.url - - @property - def keyboard(self) -> Keyboard: - """ - Get the keyboard object for the page. - - Returns: - Keyboard: The Playwright Keyboard object. - """ - return self.playwright_page.keyboard - - def _get_page(self) -> "Page": - return self - - def _get_dendrite_browser(self) -> "Dendrite": - return self.dendrite_browser - - def _get_logic_api(self) -> SyncProtocol: - return self._browser_api_client - - def goto( - self, - url: str, - timeout: Optional[float] = 30000, - wait_until: Optional[ - Literal["commit", "domcontentloaded", "load", "networkidle"] - ] = "load", - ) -> None: - """ - Navigate to a URL. - - Args: - url (str): The URL to navigate to. If no protocol is specified, 'https://' will be added. - timeout (Optional[float]): Maximum navigation time in milliseconds. - wait_until (Optional[Literal["commit", "domcontentloaded", "load", "networkidle"]]): - When to consider navigation succeeded. - """ - if not re.match("^\\w+://", url): - url = f"https://{url}" - self.playwright_page.goto(url, timeout=timeout, wait_until=wait_until) - - def get_download(self, timeout: float = 30000) -> Download: - """ - Retrieves the download event associated with. - - Args: - timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the download to complete. Defaults to 30. - - Returns: - The downloaded file data. - """ - return self.dendrite_browser._get_download(self.playwright_page, timeout) - - def _get_context(self, element: Any) -> Union[PlaywrightPage, FrameLocator]: - """ - Gets the correct context to be able to interact with an element on a different frame. - - e.g. if the element is inside an iframe, - the context will be the frame locator for that iframe. - - Args: - element (Any): The element to get the context for. - - Returns: - Union[Page, FrameLocator]: The context for the element. - """ - context = self.playwright_page - if isinstance(element, Tag): - full_path = element.get("iframe-path") - if full_path: - full_path = full_path[0] if isinstance(full_path, list) else full_path - for path in full_path.split("|"): - context = context.frame_locator(f"xpath=//iframe[@d-id='{path}']") - return context - - def scroll_to_bottom( - self, - timeout: float = 30000, - scroll_increment: int = 1000, - no_progress_limit: int = 3, - ) -> None: - """ - Scrolls to the bottom of the page until no more progress is made or a timeout occurs. - - Args: - timeout (float, optional): The maximum amount of time (in milliseconds) to continue scrolling. Defaults to 30000. - scroll_increment (int, optional): The number of pixels to scroll in each step. Defaults to 1000. - no_progress_limit (int, optional): The number of consecutive attempts with no progress before stopping. Defaults to 3. - - Returns: - None - """ - start_time = time.time() - last_scroll_position = 0 - no_progress_count = 0 - while True: - current_scroll_position = self.playwright_page.evaluate("window.scrollY") - scroll_height = self.playwright_page.evaluate("document.body.scrollHeight") - self.playwright_page.evaluate( - f"window.scrollTo(0, {current_scroll_position + scroll_increment})" - ) - if ( - self.playwright_page.viewport_size - and current_scroll_position - + self.playwright_page.viewport_size["height"] - >= scroll_height - ): - break - if current_scroll_position > last_scroll_position: - no_progress_count = 0 - else: - no_progress_count += 1 - if no_progress_count >= no_progress_limit: - break - if time.time() - start_time > timeout * 0.001: - break - last_scroll_position = current_scroll_position - time.sleep(0.1) - - def close(self) -> None: - """ - Closes the current page. - - Returns: - None - """ - self.playwright_page.close() - - def get_page_information(self, include_screenshot: bool = True) -> PageInformation: - """ - Retrieves information about the current page, including the URL, raw HTML, and a screenshot. - - Returns: - PageInformation: An object containing the page's URL, raw HTML, and a screenshot in base64 format. - """ - if include_screenshot: - base64 = self.screenshot_manager.take_full_page_screenshot() - else: - base64 = "No screenshot available" - soup = self._get_soup() - return PageInformation( - url=self.playwright_page.url, - raw_html=str(soup), - screenshot_base64=base64, - time_since_frame_navigated=self.get_time_since_last_frame_navigated(), - ) - - def _generate_dendrite_ids(self): - """ - Attempts to generate Dendrite IDs in the DOM by executing a script. - - This method will attempt to generate the Dendrite IDs up to 3 times. If all attempts fail, - an exception is raised. - - Raises: - Exception: If the Dendrite IDs could not be generated after 3 attempts. - """ - tries = 0 - while tries < 3: - try: - self.playwright_page.evaluate(GENERATE_DENDRITE_IDS_SCRIPT) - return - except Exception as e: - self.playwright_page.wait_for_load_state(state="load", timeout=3000) - logger.exception( - f"Failed to generate dendrite IDs: {e}, attempt {tries + 1}/3" - ) - tries += 1 - raise DendriteException("Failed to add d-ids to DOM.") - - def scroll_through_entire_page(self) -> None: - """ - Scrolls through the entire page. - - Returns: - None - """ - self.scroll_to_bottom() - - def upload_files( - self, - files: Union[ - str, - pathlib.Path, - FilePayload, - Sequence[Union[str, pathlib.Path]], - Sequence[FilePayload], - ], - timeout: float = 30000, - ) -> None: - """ - Uploads files to the page using a file chooser. - - Args: - files (Union[str, pathlib.Path, FilePayload, Sequence[Union[str, pathlib.Path]], Sequence[FilePayload]]): The file(s) to be uploaded. - This can be a file path, a `FilePayload` object, or a sequence of file paths or `FilePayload` objects. - timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the file chooser to be ready. Defaults to 30. - - Returns: - None - """ - file_chooser = self.dendrite_browser._get_filechooser( - self.playwright_page, timeout - ) - file_chooser.set_files(files) - - def get_content(self): - """ - Retrieves the content of the current page. - - Returns: - str: The HTML content of the current page. - """ - return self.playwright_page.content() - - def _get_soup(self) -> BeautifulSoup: - """ - Retrieves the page source as a BeautifulSoup object, with an option to exclude hidden elements. - Generates Dendrite IDs in the DOM and expands iframes. - - Returns: - BeautifulSoup: The parsed HTML of the current page. - """ - self._generate_dendrite_ids() - page_source = self.playwright_page.content() - soup = BeautifulSoup(page_source, "lxml") - self._expand_iframes(soup) - self._previous_soup = soup - return soup - - def _get_previous_soup(self) -> BeautifulSoup: - """ - Retrieves the page source generated by the latest _get_soup() call as a Beautiful soup object. If it hasn't been called yet, it will call it. - """ - if self._previous_soup is None: - return self._get_soup() - return self._previous_soup - - def _expand_iframes(self, page_source: BeautifulSoup): - """ - Expands iframes in the given page source to make their content accessible. - - Args: - page_source (BeautifulSoup): The parsed HTML content of the page. - - Returns: - None - """ - expand_iframes(self.playwright_page, page_source) - - def _get_all_elements_from_selector(self, selector: str) -> List[Element]: - dendrite_elements: List[Element] = [] - soup = self._get_soup() - elements = soup.select(selector) - for element in elements: - frame = self._get_context(element) - d_id = element.get("d-id", "") - locator = frame.locator(f"xpath=//*[@d-id='{d_id}']") - if not d_id: - continue - if isinstance(d_id, list): - d_id = d_id[0] - dendrite_elements.append( - Element(d_id, locator, self.dendrite_browser, self._browser_api_client) - ) - return dendrite_elements - - def _dump_html(self, path: str) -> None: - """ - Saves the current page's HTML content to a file. - - Args: - path (str): The file path where the HTML content should be saved. - - Returns: - None - """ - with open(path, "w") as f: - f.write(self.playwright_page.content()) - - def get_time_since_last_frame_navigated(self) -> float: - """ - Get the time elapsed since the last URL change. - - Returns: - float: The number of seconds elapsed since the last URL change. - """ - return time.time() - self._last_frame_navigated_timestamp - - def check_if_renavigated(self, initial_url: str, wait_time: float = 0.1) -> bool: - """ - Waits for a short period and checks if a main frame navigation has occurred. - - Args: - wait_time (float): The time to wait in seconds. Defaults to 0.1 seconds. - - Returns: - bool: True if a main frame navigation occurred, False otherwise. - """ - time.sleep(wait_time) - return self._last_main_frame_url != initial_url diff --git a/dendrite/browser/sync_api/_core/event_sync.py b/dendrite/browser/sync_api/_core/event_sync.py deleted file mode 100644 index 4351eee..0000000 --- a/dendrite/browser/sync_api/_core/event_sync.py +++ /dev/null @@ -1,45 +0,0 @@ -import time -import time -from typing import Generic, Optional, Type, TypeVar -from playwright.sync_api import Download, FileChooser, Page - -Events = TypeVar("Events", Download, FileChooser) -mapping = {Download: "download", FileChooser: "filechooser"} - - -class EventSync(Generic[Events]): - - def __init__(self, event_type: Type[Events]): - self.event_type = event_type - self.event_set = False - self.data: Optional[Events] = None - - def get_data(self, pw_page: Page, timeout: float = 30000) -> Events: - start_time = time.time() - while not self.event_set: - elapsed_time = (time.time() - start_time) * 1000 - if elapsed_time > timeout: - raise TimeoutError(f'Timeout waiting for event "{self.event_type}".') - pw_page.wait_for_timeout(0) - time.sleep(0.01) - data = self.data - self.data = None - self.event_set = False - if data is None: - raise ValueError("Data is None for event type: ", self.event_type) - return data - - def set_event(self, data: Events) -> None: - """ - Sets the event and stores the provided data. - - This method is used to signal that the data is ready to be retrieved by any waiting tasks. - - Args: - data (T): The data to be stored and associated with the event. - - Returns: - None - """ - self.data = data - self.event_set = True diff --git a/dendrite/browser/sync_api/_core/mixin/__init__.py b/dendrite/browser/sync_api/_core/mixin/__init__.py deleted file mode 100644 index 046a61c..0000000 --- a/dendrite/browser/sync_api/_core/mixin/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from .ask import AskMixin -from .click import ClickMixin -from .extract import ExtractionMixin -from .fill_fields import FillFieldsMixin -from .get_element import GetElementMixin -from .keyboard import KeyboardMixin -from .markdown import MarkdownMixin -from .screenshot import ScreenshotMixin -from .wait_for import WaitForMixin - -__all__ = [ - "AskMixin", - "ClickMixin", - "ExtractionMixin", - "FillFieldsMixin", - "GetElementMixin", - "KeyboardMixin", - "MarkdownMixin", - "ScreenshotMixin", - "WaitForMixin", -] diff --git a/dendrite/browser/sync_api/_core/mixin/ask.py b/dendrite/browser/sync_api/_core/mixin/ask.py deleted file mode 100644 index ea7aea4..0000000 --- a/dendrite/browser/sync_api/_core/mixin/ask.py +++ /dev/null @@ -1,189 +0,0 @@ -import time -import time -from typing import Optional, Type, overload -from loguru import logger -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.sync_api._core._type_spec import ( - JsonSchema, - PydanticModel, - TypeSpec, - convert_to_type_spec, - to_json_schema, -) -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.models.dto.ask_page_dto import AskPageDTO - -TIMEOUT_INTERVAL = [150, 450, 1000] - - -class AskMixin(DendritePageProtocol): - - @overload - def ask(self, prompt: str, type_spec: Type[str]) -> str: - """ - Asks a question about the current page and expects a response of type `str`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[str]): The expected return type, which is `str`. - - Returns: - AskPageResponse[str]: The response object containing the result of type `str`. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[bool]) -> bool: - """ - Asks a question about the current page and expects a responseof type `bool`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[bool]): The expected return type, which is `bool`. - - Returns: - AskPageResponse[bool]: The response object containing the result of type `bool`. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[int]) -> int: - """ - Asks a question about the current page and expects a response of type `int`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[int]): The expected return type, which is `int`. - - Returns: - AskPageResponse[int]: The response object containing the result of type `int`. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[float]) -> float: - """ - Asks a question about the current page and expects a response of type `float`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[float]): The expected return type, which is `float`. - - Returns: - AskPageResponse[float]: The response object containing the result of type `float`. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[PydanticModel]) -> PydanticModel: - """ - Asks a question about the current page and expects a response of a custom `PydanticModel`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[PydanticModel]): The expected return type, which is a `PydanticModel`. - - Returns: - AskPageResponse[PydanticModel]: The response object containing the result of the specified Pydantic model type. - """ - - @overload - def ask(self, prompt: str, type_spec: Type[JsonSchema]) -> JsonSchema: - """ - Asks a question about the current page and expects a response conforming to a `JsonSchema`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Type[JsonSchema]): The expected return type, which is a `JsonSchema`. - - Returns: - AskPageResponse[JsonSchema]: The response object containing the result conforming to the specified JSON schema. - """ - - @overload - def ask(self, prompt: str, type_spec: None = None) -> JsonSchema: - """ - Asks a question without specifying a type and expects a response conforming to a default `JsonSchema`. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (None, optional): The expected return type, which is `None` by default. - - Returns: - AskPageResponse[JsonSchema]: The response object containing the result conforming to the default JSON schema. - """ - - def ask( - self, prompt: str, type_spec: Optional[TypeSpec] = None, timeout: int = 15000 - ) -> TypeSpec: - """ - Asks a question and processes the response based on the specified type. - - This method sends a request to ask a question with the specified prompt and processes the response. - If a type specification is provided, the response is converted to the specified type. In case of failure, - a DendriteException is raised with relevant details. - - Args: - prompt (str): The question or prompt to be asked. - type_spec (Optional[TypeSpec], optional): The expected return type, which can be a type or a schema. Defaults to None. - - Returns: - AskPageResponse[Any]: The response object containing the result, converted to the specified type if provided. - - Raises: - DendriteException: If the request fails, the exception includes the failure message and a screenshot. - """ - start_time = time.time() - attempt_start = start_time - attempt = -1 - while True: - attempt += 1 - current_timeout = ( - TIMEOUT_INTERVAL[attempt] - if len(TIMEOUT_INTERVAL) > attempt - else TIMEOUT_INTERVAL[-1] * 1.75 - ) - elapsed_time = time.time() - start_time - remaining_time = timeout * 0.001 - elapsed_time - if remaining_time <= 0: - logger.warning( - f"Timeout reached for '{prompt}' after {attempt + 1} attempts" - ) - break - prev_attempt_time = time.time() - attempt_start - sleep_time = min( - max(current_timeout * 0.001 - prev_attempt_time, 0), remaining_time - ) - logger.debug(f"Waiting for {sleep_time} seconds before retrying") - time.sleep(sleep_time) - attempt_start = time.time() - logger.info(f"Asking '{prompt}' | Attempt {attempt + 1}") - page = self._get_page() - page_information = page.get_page_information() - schema = to_json_schema(type_spec) if type_spec else None - if elapsed_time < 5: - time_prompt = f"This page was loaded {elapsed_time} seconds ago, so it might still be loading. If the page is still loading, return failed status." - else: - time_prompt = "" - entire_prompt = prompt + time_prompt - dto = AskPageDTO( - page_information=page_information, - prompt=entire_prompt, - return_schema=schema, - ) - try: - res = self._get_logic_api().ask_page(dto) - logger.debug(f"Got response in {time.time() - attempt_start} seconds") - if res.status == "error": - logger.warning( - f"Error response on attempt {attempt + 1}: {res.return_data}" - ) - continue - converted_res = res.return_data - if type_spec is not None: - converted_res = convert_to_type_spec(type_spec, res.return_data) - return converted_res - except Exception as e: - logger.error(f"Exception occurred on attempt {attempt + 1}: {str(e)}") - if attempt == len(TIMEOUT_INTERVAL) - 1: - raise - raise DendriteException( - message=f"Failed to get response for '{prompt}' after {attempt + 1} attempts", - screenshot_base64=page_information.screenshot_base64, - ) diff --git a/dendrite/browser/sync_api/_core/mixin/click.py b/dendrite/browser/sync_api/_core/mixin/click.py deleted file mode 100644 index 0b6adaa..0000000 --- a/dendrite/browser/sync_api/_core/mixin/click.py +++ /dev/null @@ -1,56 +0,0 @@ -from typing import Optional -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.models.response.interaction_response import InteractionResponse - - -class ClickMixin(GetElementMixin, DendritePageProtocol): - - def click( - self, - prompt: str, - expected_outcome: Optional[str] = None, - use_cache: bool = True, - timeout: int = 15000, - force: bool = False, - *args, - **kwargs, - ) -> InteractionResponse: - """ - Clicks an element on the page based on the provided prompt. - - This method combines the functionality of get_element and click, - allowing for a more concise way to interact with elements on the page. - - Args: - prompt (str): The prompt describing the element to be clicked. - expected_outcome (Optional[str]): The expected outcome of the click action. - use_cache (bool, optional): Whether to use cached results for element retrieval. Defaults to True. - timeout (int, optional): The timeout (in milliseconds) for the click operation. Defaults to 15000. - force (bool, optional): Whether to force the click operation. Defaults to False. - *args: Additional positional arguments for the click operation. - **kwargs: Additional keyword arguments for the click operation. - - Returns: - InteractionResponse: The response from the interaction. - - Raises: - DendriteException: If no suitable element is found or if the click operation fails. - """ - augmented_prompt = prompt + "\n\nThe element should be clickable." - element = self.get_element( - augmented_prompt, use_cache=use_cache, timeout=timeout - ) - if not element: - raise DendriteException( - message=f"No element found with the prompt: {prompt}", - screenshot_base64="", - ) - return element.click( - *args, - expected_outcome=expected_outcome, - timeout=timeout, - force=force, - **kwargs, - ) diff --git a/dendrite/browser/sync_api/_core/mixin/extract.py b/dendrite/browser/sync_api/_core/mixin/extract.py deleted file mode 100644 index f704d4a..0000000 --- a/dendrite/browser/sync_api/_core/mixin/extract.py +++ /dev/null @@ -1,284 +0,0 @@ -import time -import time -from typing import Any, Callable, List, Optional, Type, overload -from loguru import logger -from dendrite.browser.sync_api._core._managers.navigation_tracker import ( - NavigationTracker, -) -from dendrite.browser.sync_api._core._type_spec import ( - JsonSchema, - PydanticModel, - TypeSpec, - convert_to_type_spec, - to_json_schema, -) -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.logic.code.code_session import execute -from dendrite.models.dto.cached_extract_dto import CachedExtractDTO -from dendrite.models.dto.extract_dto import ExtractDTO -from dendrite.models.response.extract_response import ExtractResponse -from dendrite.models.scripts import Script - -CACHE_TIMEOUT = 5 - - -class ExtractionMixin(DendritePageProtocol): - """ - Mixin that provides extraction functionality for web pages. - - This mixin provides various `extract` methods that allow extracting - different types of data (e.g., bool, int, float, string, Pydantic models, etc.) - from a web page based on a given prompt. - """ - - @overload - def extract( - self, - prompt: str, - type_spec: Type[bool], - use_cache: bool = True, - timeout: int = 180, - ) -> bool: ... - - @overload - def extract( - self, - prompt: str, - type_spec: Type[int], - use_cache: bool = True, - timeout: int = 180, - ) -> int: ... - - @overload - def extract( - self, - prompt: str, - type_spec: Type[float], - use_cache: bool = True, - timeout: int = 180, - ) -> float: ... - - @overload - def extract( - self, - prompt: str, - type_spec: Type[str], - use_cache: bool = True, - timeout: int = 180, - ) -> str: ... - - @overload - def extract( - self, - prompt: Optional[str], - type_spec: Type[PydanticModel], - use_cache: bool = True, - timeout: int = 180, - ) -> PydanticModel: ... - - @overload - def extract( - self, - prompt: Optional[str], - type_spec: JsonSchema, - use_cache: bool = True, - timeout: int = 180, - ) -> JsonSchema: ... - - @overload - def extract( - self, - prompt: str, - type_spec: None = None, - use_cache: bool = True, - timeout: int = 180, - ) -> Any: ... - - def extract( - self, - prompt: Optional[str], - type_spec: Optional[TypeSpec] = None, - use_cache: bool = True, - timeout: int = 180, - ) -> TypeSpec: - """ - Extract data from a web page based on a prompt and optional type specification. - Args: - prompt (Optional[str]): The prompt to describe the information to extract. - type_spec (Optional[TypeSpec], optional): The type specification for the extracted data. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached scripts before falling back to the - extraction agent for the remaining time that will attempt to generate a new script. Defaults to 15000 (15 seconds). - - Returns: - ExtractResponse: The extracted data wrapped in a ExtractResponse object. - Raises: - TimeoutError: If the extraction process exceeds the specified timeout. - """ - logger.info(f"Starting extraction with prompt: {prompt}") - json_schema = None - if type_spec: - json_schema = to_json_schema(type_spec) - logger.debug(f"Type specification converted to JSON schema: {json_schema}") - if prompt is None: - prompt = "" - start_time = time.time() - page = self._get_page() - navigation_tracker = NavigationTracker(page) - navigation_tracker.start_nav_tracking() - if use_cache: - logger.info("Testing cache") - cached_result = self._try_cached_extraction(prompt, json_schema) - if cached_result: - return convert_and_return_result(cached_result, type_spec) - logger.info( - "Using extraction agent to perform extraction, since no cache was found or failed." - ) - result = self._extract_with_agent( - prompt, json_schema, timeout - (time.time() - start_time) - ) - if result: - return convert_and_return_result(result, type_spec) - logger.error(f"Extraction failed after {time.time() - start_time:.2f} seconds") - return None - - def _try_cached_extraction( - self, prompt: str, json_schema: Optional[JsonSchema] - ) -> Optional[ExtractResponse]: - """ - Attempts to extract data using cached scripts with exponential backoff. - - Args: - prompt: The prompt describing what to extract - json_schema: Optional JSON schema for type validation - - Returns: - ExtractResponse if successful, None otherwise - """ - page = self._get_page() - dto = CachedExtractDTO(url=page.url, prompt=prompt) - scripts = self._get_logic_api().get_cached_scripts(dto) - logger.debug(f"Found {len(scripts)} scripts in cache, {scripts}") - if len(scripts) == 0: - logger.debug( - f"No scripts found in cache for prompt: {prompt} in domain: {page.url}" - ) - return None - - def try_cached_extract(): - page = self._get_page() - soup = page._get_soup() - for script in scripts: - res = test_script(script, str(soup), json_schema) - if res is not None: - return ExtractResponse( - status="success", - message="Re-used a preexisting script from cache with the same specifications.", - return_data=res, - created_script=script.script, - ) - return None - - return _attempt_with_backoff_helper( - "cached_extraction", try_cached_extract, CACHE_TIMEOUT - ) - - def _extract_with_agent( - self, prompt: str, json_schema: Optional[JsonSchema], remaining_timeout: float - ) -> Optional[ExtractResponse]: - """ - Attempts to extract data using the extraction agent with exponential backoff. - - Args: - prompt: The prompt describing what to extract - json_schema: Optional JSON schema for type validation - remaining_timeout: Maximum time to spend on extraction - - Returns: - ExtractResponse if successful, None otherwise - """ - - def try_extract_with_agent(): - page = self._get_page() - page_information = page.get_page_information(include_screenshot=True) - extract_dto = ExtractDTO( - page_information=page_information, - prompt=prompt, - return_data_json_schema=json_schema, - use_screenshot=True, - ) - res: ExtractResponse = self._get_logic_api().extract(extract_dto) - if res.status == "impossible": - logger.error(f"Impossible to extract data. Reason: {res.message}") - return None - if res.status == "success": - logger.success(f"Extraction successful: '{res.message}'") - return res - return None - - return _attempt_with_backoff_helper( - "extraction_agent", try_extract_with_agent, remaining_timeout - ) - - -def _attempt_with_backoff_helper( - operation_name: str, - operation: Callable, - timeout: float, - backoff_intervals: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0], -) -> Optional[Any]: - """ - Generic helper function that implements exponential backoff for operations. - - Args: - operation_name: Name of the operation for logging - operation: Async function to execute - timeout: Maximum time to spend attempting the operation - backoff_intervals: List of timeouts between attempts - - Returns: - The result of the operation if successful, None otherwise - """ - total_elapsed_time = 0 - start_time = time.time() - for i, current_timeout in enumerate(backoff_intervals): - if total_elapsed_time >= timeout: - logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") - return None - request_start_time = time.time() - result = operation() - request_duration = time.time() - request_start_time - if result: - return result - sleep_duration = max(0, current_timeout - request_duration) - logger.info( - f"{operation_name} attempt {i + 1} failed. Sleeping for {sleep_duration:.2f} seconds" - ) - time.sleep(sleep_duration) - total_elapsed_time = time.time() - start_time - logger.error( - f"All {operation_name} attempts failed after {total_elapsed_time:.2f} seconds" - ) - return None - - -def convert_and_return_result( - res: ExtractResponse, type_spec: Optional[TypeSpec] -) -> TypeSpec: - converted_res = res.return_data - if type_spec is not None: - logger.debug("Converting extraction result to specified type") - converted_res = convert_to_type_spec(type_spec, res.return_data) - logger.info("Extraction process completed successfully") - return converted_res - - -def test_script( - script: Script, raw_html: str, return_data_json_schema: Any -) -> Optional[Any]: - try: - res = execute(script.script, raw_html, return_data_json_schema) - return res - except Exception as e: - logger.debug(f"Script failed with error: {str(e)} ") diff --git a/dendrite/browser/sync_api/_core/mixin/fill_fields.py b/dendrite/browser/sync_api/_core/mixin/fill_fields.py deleted file mode 100644 index e49825b..0000000 --- a/dendrite/browser/sync_api/_core/mixin/fill_fields.py +++ /dev/null @@ -1,76 +0,0 @@ -import time -from typing import Any, Dict, Optional -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.sync_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.models.response.interaction_response import InteractionResponse - - -class FillFieldsMixin(GetElementMixin, DendritePageProtocol): - - def fill_fields(self, fields: Dict[str, Any]): - """ - Fills multiple fields on the page with the provided values. - - This method iterates through the given dictionary of fields and their corresponding values, - making a separate fill request for each key-value pair. - - Args: - fields (Dict[str, Any]): A dictionary where each key is a field identifier (e.g., a prompt or selector) - and each value is the content to fill in that field. - - Returns: - None - - Note: - This method will make multiple fill requests, one for each key in the 'fields' dictionary. - """ - for field, value in fields.items(): - prompt = f"I'll be filling in text in several fields with these keys: {fields.keys()} in this page. Get the field best described as '{field}'. I want to fill it with a '{type(value)}' type value." - self.fill(prompt, value) - time.sleep(0.5) - - def fill( - self, - prompt: str, - value: str, - expected_outcome: Optional[str] = None, - use_cache: bool = True, - timeout: int = 15000, - *args, - kwargs={}, - ) -> InteractionResponse: - """ - Fills an element on the page with the provided value based on the given prompt. - - This method combines the functionality of get_element and fill, - allowing for a more concise way to interact with elements on the page. - - Args: - prompt (str): The prompt describing the element to be filled. - value (str): The value to fill the element with. - expected_outcome (Optional[str]): The expected outcome of the fill action. - use_cache (bool, optional): Whether to use cached results for element retrieval. Defaults to True. - max_retries (int, optional): The maximum number of retry attempts for element retrieval. Defaults to 3. - timeout (int, optional): The timeout (in milliseconds) for the fill operation. Defaults to 15000. - *args: Additional positional arguments for the fill operation. - kwargs: Additional keyword arguments for the fill operation. - - Returns: - InteractionResponse: The response from the interaction. - - Raises: - DendriteException: If no suitable element is found or if the fill operation fails. - """ - augmented_prompt = prompt + "\n\nMake sure the element can be filled with text." - element = self.get_element( - augmented_prompt, use_cache=use_cache, timeout=timeout - ) - if not element: - raise DendriteException( - message=f"No element found with the prompt: {prompt}", - screenshot_base64="", - ) - return element.fill( - value, *args, expected_outcome=expected_outcome, timeout=timeout, **kwargs - ) diff --git a/dendrite/browser/sync_api/_core/mixin/get_element.py b/dendrite/browser/sync_api/_core/mixin/get_element.py deleted file mode 100644 index 22b7583..0000000 --- a/dendrite/browser/sync_api/_core/mixin/get_element.py +++ /dev/null @@ -1,336 +0,0 @@ -import time -import time -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - Literal, - Optional, - Union, - overload, -) -from bs4 import BeautifulSoup -from loguru import logger -from dendrite.browser.sync_api._core._utils import _get_all_elements_from_selector_soup -from dendrite.browser.sync_api._core.dendrite_element import Element - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_page import Page -from dendrite.browser.sync_api._core.models.response import ElementsResponse -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol -from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO -from dendrite.models.dto.get_elements_dto import GetElementsDTO - -CACHE_TIMEOUT = 5 - - -class GetElementMixin(DendritePageProtocol): - - @overload - def get_elements( - self, - prompt_or_elements: str, - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> List[Element]: - """ - Retrieves a list of Dendrite elements based on a string prompt. - - Args: - prompt_or_elements (str): The prompt describing the elements to be retrieved. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - List[Element]: A list of Dendrite elements found on the page. - """ - - @overload - def get_elements( - self, - prompt_or_elements: Dict[str, str], - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> ElementsResponse: - """ - Retrieves Dendrite elements based on a dictionary. - - Args: - prompt_or_elements (Dict[str, str]): A dictionary where keys are field names and values are prompts describing the elements to be retrieved. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - ElementsResponse: A response object containing the retrieved elements with attributes matching the keys in the dict. - """ - - def get_elements( - self, - prompt_or_elements: Union[str, Dict[str, str]], - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> Union[List[Element], ElementsResponse]: - """ - Retrieves Dendrite elements based on either a string prompt or a dictionary of prompts. - - This method determines the type of the input (string or dictionary) and retrieves the appropriate elements. - If the input is a string, it fetches a list of elements. If the input is a dictionary, it fetches elements for each key-value pair. - - Args: - prompt_or_elements (Union[str, Dict[str, str]]): The prompt or dictionary of prompts for element retrieval. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - Union[List[Element], ElementsResponse]: A list of elements or a response object containing the retrieved elements. - - Raises: - ValueError: If the input is neither a string nor a dictionary. - """ - return self._get_element( - prompt_or_elements, - only_one=False, - use_cache=use_cache, - timeout=timeout / 1000, - ) - - def get_element( - self, prompt: str, use_cache=True, timeout=15000 - ) -> Optional[Element]: - """ - Retrieves a single Dendrite element based on the provided prompt. - - Args: - prompt (str): The prompt describing the element to be retrieved. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - - Returns: - Element: The retrieved element. - """ - return self._get_element( - prompt, only_one=True, use_cache=use_cache, timeout=timeout / 1000 - ) - - @overload - def _get_element( - self, prompt_or_elements: str, only_one: Literal[True], use_cache: bool, timeout - ) -> Optional[Element]: - """ - Retrieves a single Dendrite element based on the provided prompt. - - Args: - prompt (Union[str, Dict[str, str]]): The prompt describing the element to be retrieved. - only_one (Literal[True]): Indicates that only one element should be retrieved. - use_cache (bool): Whether to use cached results. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - - Returns: - Element: The retrieved element. - """ - - @overload - def _get_element( - self, - prompt_or_elements: Union[str, Dict[str, str]], - only_one: Literal[False], - use_cache: bool, - timeout, - ) -> Union[List[Element], ElementsResponse]: - """ - Retrieves a list of Dendrite elements based on the provided prompt. - - Args: - prompt (str): The prompt describing the elements to be retrieved. - only_one (Literal[False]): Indicates that multiple elements should be retrieved. - use_cache (bool): Whether to use cached results. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - - Returns: - List[Element]: A list of retrieved elements. - """ - - def _get_element( - self, - prompt_or_elements: Union[str, Dict[str, str]], - only_one: bool, - use_cache: bool, - timeout: float, - ) -> Union[Optional[Element], List[Element], ElementsResponse]: - """ - Retrieves Dendrite elements based on the provided prompt, either a single element or a list of elements. - - This method sends a request with the prompt and retrieves the elements based on the `only_one` flag. - - Args: - prompt_or_elements (Union[str, Dict[str, str]]): The prompt or dictionary of prompts for element retrieval. - only_one (bool): Whether to retrieve only one element or a list of elements. - use_cache (bool): Whether to use cached results. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - - Returns: - Union[Element, List[Element], ElementsResponse]: The retrieved element, list of elements, or response object. - """ - if isinstance(prompt_or_elements, Dict): - return None - start_time = time.time() - page = self._get_page() - soup = page._get_soup() - if use_cache: - cached_elements = self._try_cached_selectors( - page, soup, prompt_or_elements, only_one - ) - if cached_elements: - return cached_elements - logger.info( - "Proceeding to use the find element agent to find the requested elements." - ) - res = try_get_element( - self, - prompt_or_elements, - only_one, - remaining_timeout=timeout - (time.time() - start_time), - ) - if res: - return res - logger.error( - f"Failed to retrieve elements within the specified timeout of {timeout} seconds" - ) - return None - - def _try_cached_selectors( - self, page: "Page", soup: BeautifulSoup, prompt: str, only_one: bool - ) -> Union[Optional[Element], List[Element]]: - """ - Attempts to retrieve elements using cached selectors with exponential backoff. - - Args: - page: The current page object - soup: The BeautifulSoup object of the current page - prompt: The prompt to search for - only_one: Whether to return only one element - - Returns: - The found elements if successful, None otherwise - """ - dto = CachedSelectorDTO(url=page.url, prompt=prompt) - selectors = self._get_logic_api().get_cached_selectors(dto) - if len(selectors) == 0: - logger.debug("No cached selectors found") - return None - logger.debug("Attempting to use cached selectors with backoff") - str_selectors = list(map(lambda x: x.selector, selectors)) - - def try_cached_selectors(): - return get_elements_from_selectors_soup(page, soup, str_selectors, only_one) - - return _attempt_with_backoff_helper( - "cached_selectors", try_cached_selectors, timeout=CACHE_TIMEOUT - ) - - -def _attempt_with_backoff_helper( - operation_name: str, - operation: Callable, - timeout: float, - backoff_intervals: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0], -) -> Optional[Any]: - """ - Generic helper function that implements exponential backoff for operations. - - Args: - operation_name: Name of the operation for logging - operation: Async function to execute - timeout: Maximum time to spend attempting the operation - backoff_intervals: List of timeouts between attempts - - Returns: - The result of the operation if successful, None otherwise - """ - total_elapsed_time = 0 - start_time = time.time() - for i, current_timeout in enumerate(backoff_intervals): - if total_elapsed_time >= timeout: - logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") - return None - request_start_time = time.time() - result = operation() - request_duration = time.time() - request_start_time - if result: - return result - sleep_duration = max(0, current_timeout - request_duration) - logger.info( - f"{operation_name} attempt {i + 1} failed. Sleeping for {sleep_duration:.2f} seconds" - ) - time.sleep(sleep_duration) - total_elapsed_time = time.time() - start_time - logger.error( - f"All {operation_name} attempts failed after {total_elapsed_time:.2f} seconds" - ) - return None - - -def try_get_element( - obj: DendritePageProtocol, - prompt_or_elements: Union[str, Dict[str, str]], - only_one: bool, - remaining_timeout: float, -) -> Union[Optional[Element], List[Element], ElementsResponse]: - - def _try_get_element(): - page = obj._get_page() - page_information = page.get_page_information() - dto = GetElementsDTO( - page_information=page_information, - prompt=prompt_or_elements, - only_one=only_one, - ) - res = obj._get_logic_api().get_element(dto) - if res.status == "impossible": - logger.error( - f"Impossible to get elements for '{prompt_or_elements}'. Reason: {res.message}" - ) - return None - if res.status == "success": - logger.success(f"d[id]: {res.d_id} Selectors:{res.selectors}") - if res.selectors is not None: - return get_elements_from_selectors_soup( - page, page._get_previous_soup(), res.selectors, only_one - ) - return None - - return _attempt_with_backoff_helper( - "find_element_agent", _try_get_element, remaining_timeout - ) - - -def get_elements_from_selectors_soup( - page: "Page", soup: BeautifulSoup, selectors: List[str], only_one: bool -) -> Union[Optional[Element], List[Element]]: - for selector in reversed(selectors): - dendrite_elements = _get_all_elements_from_selector_soup(selector, soup, page) - if len(dendrite_elements) > 0: - return dendrite_elements[0] if only_one else dendrite_elements - return None diff --git a/dendrite/browser/sync_api/_core/mixin/keyboard.py b/dendrite/browser/sync_api/_core/mixin/keyboard.py deleted file mode 100644 index 1ab7894..0000000 --- a/dendrite/browser/sync_api/_core/mixin/keyboard.py +++ /dev/null @@ -1,62 +0,0 @@ -from typing import Literal, Union -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol - - -class KeyboardMixin(DendritePageProtocol): - - def press( - self, - key: Union[ - str, - Literal[ - "Enter", - "Tab", - "Escape", - "Backspace", - "ArrowUp", - "ArrowDown", - "ArrowLeft", - "ArrowRight", - ], - ], - hold_shift: bool = False, - hold_ctrl: bool = False, - hold_alt: bool = False, - hold_cmd: bool = False, - ): - """ - Presses a keyboard key on the active page, optionally with modifier keys. - - Args: - key (Union[str, Literal["Enter", "Tab", "Escape", "Backspace", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight"]]): The main key to be pressed. - hold_shift (bool, optional): Whether to hold the Shift key. Defaults to False. - hold_ctrl (bool, optional): Whether to hold the Control key. Defaults to False. - hold_alt (bool, optional): Whether to hold the Alt key. Defaults to False. - hold_cmd (bool, optional): Whether to hold the Command key (Meta on some systems). Defaults to False. - - Returns: - Any: The result of the key press operation. - - Raises: - DendriteException: If the key press operation fails. - """ - modifiers = [] - if hold_shift: - modifiers.append("Shift") - if hold_ctrl: - modifiers.append("Control") - if hold_alt: - modifiers.append("Alt") - if hold_cmd: - modifiers.append("Meta") - if modifiers: - key = "+".join(modifiers + [key]) - try: - page = self._get_page() - page.keyboard.press(key) - except Exception as e: - raise DendriteException( - message=f"Failed to press key: {key}. Error: {str(e)}", - screenshot_base64="", - ) diff --git a/dendrite/browser/sync_api/_core/mixin/markdown.py b/dendrite/browser/sync_api/_core/mixin/markdown.py deleted file mode 100644 index fce98d2..0000000 --- a/dendrite/browser/sync_api/_core/mixin/markdown.py +++ /dev/null @@ -1,23 +0,0 @@ -import re -from typing import Optional -from bs4 import BeautifulSoup -from markdownify import markdownify as md -from dendrite.browser.sync_api._core.mixin.extract import ExtractionMixin -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol - - -class MarkdownMixin(ExtractionMixin, DendritePageProtocol): - - def markdown(self, prompt: Optional[str] = None): - page = self._get_page() - page_information = page.get_page_information() - if prompt: - extract_prompt = f"Create a script that returns the HTML from one element from the DOM that best matches this requested section of the website.\n\nDescription of section: '{prompt}'\n\nWe will be converting your returned HTML to markdown, so just return ONE stringified HTML element and nothing else. It's OK if extra information is present. Example script: 'response_data = soup.find('tag', {{'attribute': 'value'}}).prettify()'" - res = self.extract(extract_prompt) - markdown_text = md(res) - cleaned_markdown = re.sub("\\n{3,}", "\n\n", markdown_text) - return cleaned_markdown - else: - markdown_text = md(page_information.raw_html) - cleaned_markdown = re.sub("\\n{3,}", "\n\n", markdown_text) - return cleaned_markdown diff --git a/dendrite/browser/sync_api/_core/mixin/screenshot.py b/dendrite/browser/sync_api/_core/mixin/screenshot.py deleted file mode 100644 index bd3fab2..0000000 --- a/dendrite/browser/sync_api/_core/mixin/screenshot.py +++ /dev/null @@ -1,20 +0,0 @@ -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol - - -class ScreenshotMixin(DendritePageProtocol): - - def screenshot(self, full_page: bool = False) -> str: - """ - Take a screenshot of the current page. - - Args: - full_page (bool, optional): If True, captures the full page. If False, captures only the viewport. Defaults to False. - - Returns: - str: A base64 encoded string of the screenshot in JPEG format. - """ - page = self._get_page() - if full_page: - return page.screenshot_manager.take_full_page_screenshot() - else: - return page.screenshot_manager.take_viewport_screenshot() diff --git a/dendrite/browser/sync_api/_core/mixin/wait_for.py b/dendrite/browser/sync_api/_core/mixin/wait_for.py deleted file mode 100644 index df29d12..0000000 --- a/dendrite/browser/sync_api/_core/mixin/wait_for.py +++ /dev/null @@ -1,53 +0,0 @@ -import time -import time -from loguru import logger -from dendrite.browser._common._exceptions.dendrite_exception import ( - DendriteException, - PageConditionNotMet, -) -from dendrite.browser.sync_api._core.mixin.ask import AskMixin -from dendrite.browser.sync_api._core.protocol.page_protocol import DendritePageProtocol - - -class WaitForMixin(AskMixin, DendritePageProtocol): - - def wait_for(self, prompt: str, timeout: float = 30000): - """ - Waits for the condition specified in the prompt to become true by periodically checking the page content. - - This method attempts to retrieve the page information and evaluate whether the specified - condition (provided in the prompt) is met. It continues to retry until the total elapsed time - exceeds the specified timeout. - - Args: - prompt (str): The prompt to determine the condition to wait for on the page. - timeout (float, optional): The maximum time (in milliseconds) to wait for the condition. Defaults to 15000. - - Returns: - Any: The result of the condition evaluation if successful. - - Raises: - PageConditionNotMet: If the condition is not met within the specified timeout. - """ - start_time = time.time() - time.sleep(0.2) - while True: - elapsed_time = (time.time() - start_time) * 1000 - if elapsed_time >= timeout: - break - page = self._get_page() - page_information = page.get_page_information() - prompt_with_instruction = f"Prompt: '{prompt}'\n\nReturn a boolean that determines if the requested information or thing is available on the page. {round(page_information.time_since_frame_navigated, 2)} seconds have passed since the page first loaded." - try: - res = self.ask(prompt_with_instruction, bool) - if res: - return res - except DendriteException as e: - logger.debug(f"Attempt failed: {e.message}") - time.sleep(0.5) - page = self._get_page() - page_information = page.get_page_information() - raise PageConditionNotMet( - message=f"Failed to wait for the requested condition within the {timeout}ms timeout.", - screenshot_base64=page_information.screenshot_base64, - ) diff --git a/dendrite/browser/sync_api/_core/models/__init__.py b/dendrite/browser/sync_api/_core/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_core/models/authentication.py b/dendrite/browser/sync_api/_core/models/authentication.py deleted file mode 100644 index 250a8ce..0000000 --- a/dendrite/browser/sync_api/_core/models/authentication.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import List, Literal, Optional -from pydantic import BaseModel -from typing_extensions import TypedDict - - -class Cookie(TypedDict, total=False): - name: str - value: str - domain: str - path: str - expires: float - httpOnly: bool - secure: bool - sameSite: Literal["Lax", "None", "Strict"] - - -class LocalStorageEntry(TypedDict): - name: str - value: str - - -class OriginState(TypedDict): - origin: str - localStorage: List[LocalStorageEntry] - - -class StorageState(TypedDict, total=False): - cookies: List[Cookie] - origins: List[OriginState] - - -class DomainState(BaseModel): - domain: str - storage_state: StorageState - - -class AuthSession(BaseModel): - user_agent: Optional[str] - domain_states: List[DomainState] - - def to_storage_state(self) -> StorageState: - cookies = [] - origins = [] - for domain_state in self.domain_states: - cookies.extend(domain_state.storage_state.get("cookies", [])) - origins.extend(domain_state.storage_state.get("origins", [])) - return StorageState(cookies=cookies, origins=origins) diff --git a/dendrite/browser/sync_api/_core/models/download_interface.py b/dendrite/browser/sync_api/_core/models/download_interface.py deleted file mode 100644 index 8a68843..0000000 --- a/dendrite/browser/sync_api/_core/models/download_interface.py +++ /dev/null @@ -1,20 +0,0 @@ -from abc import ABC, abstractmethod -from pathlib import Path -from typing import Any, Union -from playwright.sync_api import Download - - -class DownloadInterface(ABC, Download): - - def __init__(self, download: Download): - self._download = download - - def __getattribute__(self, name: str) -> Any: - try: - return super().__getattribute__(name) - except AttributeError: - return getattr(self._download, name) - - @abstractmethod - def save_as(self, path: Union[str, Path]) -> None: - pass diff --git a/dendrite/browser/sync_api/_core/models/page_diff_information.py b/dendrite/browser/sync_api/_core/models/page_diff_information.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_core/models/page_information.py b/dendrite/browser/sync_api/_core/models/page_information.py deleted file mode 100644 index 788dc87..0000000 --- a/dendrite/browser/sync_api/_core/models/page_information.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Optional -from pydantic import BaseModel -from typing_extensions import TypedDict - - -class InteractableElementInfo(TypedDict): - attrs: Optional[str] - text: Optional[str] - - -class PageInformation(BaseModel): - url: str - raw_html: str - screenshot_base64: str - time_since_frame_navigated: float diff --git a/dendrite/browser/sync_api/_core/models/response.py b/dendrite/browser/sync_api/_core/models/response.py deleted file mode 100644 index 50d9a23..0000000 --- a/dendrite/browser/sync_api/_core/models/response.py +++ /dev/null @@ -1,54 +0,0 @@ -from typing import Dict, Iterator -from dendrite.browser.sync_api._core.dendrite_element import Element - - -class ElementsResponse: - """ - ElementsResponse is a class that encapsulates a dictionary of Dendrite elements, - allowing for attribute-style access and other convenient interactions. - - This class is used to store and access the elements retrieved by the `get_elements` function. - The attributes of this class dynamically match the keys of the dictionary passed to the `get_elements` function, - allowing for direct attribute-style access to the corresponding `Element` objects. - - Attributes: - _data (Dict[str, Element]): A dictionary where keys are the names of elements and values are the corresponding `Element` objects. - - Args: - data (Dict[str, Element]): The dictionary of elements to be encapsulated by the class. - - Methods: - __getattr__(name: str) -> Element: - Allows attribute-style access to the elements in the dictionary. - - __getitem__(key: str) -> Element: - Enables dictionary-style access to the elements. - - __iter__() -> Iterator[str]: - Provides an iterator over the keys in the dictionary. - - __repr__() -> str: - Returns a string representation of the class instance. - """ - - _data: Dict[str, Element] - - def __init__(self, data: Dict[str, Element]): - self._data = data - - def __getattr__(self, name: str) -> Element: - try: - return self._data[name] - except KeyError: - raise AttributeError( - f"'{self.__class__.__name__}' object has no attribute '{name}'" - ) - - def __getitem__(self, key: str) -> Element: - return self._data[key] - - def __iter__(self) -> Iterator[str]: - return iter(self._data) - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self._data})" diff --git a/dendrite/browser/sync_api/_core/protocol/page_protocol.py b/dendrite/browser/sync_api/_core/protocol/page_protocol.py deleted file mode 100644 index b37969e..0000000 --- a/dendrite/browser/sync_api/_core/protocol/page_protocol.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import TYPE_CHECKING, Protocol -from dendrite.logic.interfaces import SyncProtocol - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite - from dendrite.browser.sync_api._core.dendrite_page import Page - - -class DendritePageProtocol(Protocol): - """ - Protocol that specifies the required methods and attributes - for the `ExtractionMixin` to work. - """ - - def _get_dendrite_browser(self) -> "Dendrite": ... - - def _get_logic_api(self) -> SyncProtocol: ... - - def _get_page(self) -> "Page": ... diff --git a/dendrite/browser/sync_api/_dom/__init__.py b/dendrite/browser/sync_api/_dom/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_remote_impl/__init__.py b/dendrite/browser/sync_api/_remote_impl/__init__.py deleted file mode 100644 index 4d00d3c..0000000 --- a/dendrite/browser/sync_api/_remote_impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .browserbase import BrowserbaseDownload - -__all__ = ["BrowserbaseDownload"] diff --git a/dendrite/browser/sync_api/_remote_impl/browserbase/__init__.py b/dendrite/browser/sync_api/_remote_impl/browserbase/__init__.py deleted file mode 100644 index eb977c7..0000000 --- a/dendrite/browser/sync_api/_remote_impl/browserbase/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._download import BrowserbaseDownload - -__all__ = ["BrowserbaseDownload"] diff --git a/dendrite/browser/sync_api/_remote_impl/browserbase/_client.py b/dendrite/browser/sync_api/_remote_impl/browserbase/_client.py deleted file mode 100644 index ddc6831..0000000 --- a/dendrite/browser/sync_api/_remote_impl/browserbase/_client.py +++ /dev/null @@ -1,63 +0,0 @@ -import time -import time -from pathlib import Path -from typing import Optional, Union -import httpx -from loguru import logger -from dendrite.browser._common._exceptions.dendrite_exception import DendriteException - - -class BrowserbaseClient: - - def __init__(self, api_key: str, project_id: str) -> None: - self.api_key = api_key - self.project_id = project_id - - def create_session(self) -> str: - logger.debug("Creating session") - "\n Creates a session using the Browserbase API.\n\n Returns:\n str: The ID of the created session.\n " - url = "https://www.browserbase.com/v1/sessions" - headers = {"Content-Type": "application/json", "x-bb-api-key": self.api_key} - json = {"projectId": self.project_id, "keepAlive": False} - response = httpx.post(url, json=json, headers=headers) - if response.status_code >= 400: - raise DendriteException(f"Failed to create session: {response.text}") - return response.json()["id"] - - def stop_session(self, session_id: str): - url = f"https://www.browserbase.com/v1/sessions/{session_id}" - headers = {"Content-Type": "application/json", "x-bb-api-key": self.api_key} - json = {"projectId": self.project_id, "status": "REQUEST_RELEASE"} - with httpx.Client() as client: - response = client.post(url, json=json, headers=headers) - return response.json() - - def connect_url(self, enable_proxy: bool, session_id: Optional[str] = None) -> str: - url = f"wss://connect.browserbase.com?apiKey={self.api_key}" - if session_id: - url += f"&sessionId={session_id}" - if enable_proxy: - url += "&enableProxy=true" - return url - - def save_downloads_on_disk( - self, session_id: str, path: Union[str, Path], retry_for_seconds: float - ): - url = f"https://www.browserbase.com/v1/sessions/{session_id}/downloads" - headers = {"x-bb-api-key": self.api_key} - file_path = Path(path) - with httpx.Client() as session: - timeout = time.time() + retry_for_seconds - while time.time() < timeout: - try: - response = session.get(url, headers=headers) - if response.status_code == 200: - array_buffer = response.read() - if len(array_buffer) > 0: - with open(file_path, "wb") as f: - f.write(array_buffer) - return - except Exception as e: - logger.debug(f"Error fetching downloads: {e}") - time.sleep(2) - logger.debug("Failed to download files within the time limit.") diff --git a/dendrite/browser/sync_api/_remote_impl/browserbase/_download.py b/dendrite/browser/sync_api/_remote_impl/browserbase/_download.py deleted file mode 100644 index 24647c4..0000000 --- a/dendrite/browser/sync_api/_remote_impl/browserbase/_download.py +++ /dev/null @@ -1,53 +0,0 @@ -import re -import shutil -import zipfile -from pathlib import Path -from typing import Union -from loguru import logger -from playwright.sync_api import Download -from dendrite.browser.sync_api._core.models.download_interface import DownloadInterface -from dendrite.browser.sync_api._remote_impl.browserbase._client import BrowserbaseClient - - -class BrowserbaseDownload(DownloadInterface): - - def __init__( - self, session_id: str, download: Download, client: BrowserbaseClient - ) -> None: - super().__init__(download) - self._session_id = session_id - self._client = client - - def save_as(self, path: Union[str, Path], timeout: float = 20) -> None: - """ - Save the latest file from the downloaded ZIP archive to the specified path. - - Args: - path (Union[str, Path]): The destination file path where the latest file will be saved. - timeout (float, optional): Timeout for the save operation. Defaults to 20 seconds. - - Raises: - Exception: If no matching files are found in the ZIP archive or if the file cannot be saved. - """ - destination_path = Path(path) - source_path = self._download.path() - destination_path.parent.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(source_path, "r") as zip_ref: - file_list = zip_ref.namelist() - sorted_files = sorted(file_list, key=extract_timestamp, reverse=True) - if not sorted_files: - raise FileNotFoundError( - "No files found in the Browserbase download ZIP" - ) - latest_file = sorted_files[0] - with zip_ref.open(latest_file) as source, open( - destination_path, "wb" - ) as target: - shutil.copyfileobj(source, target) - logger.info(f"Latest file saved successfully to {destination_path}") - - -def extract_timestamp(filename): - timestamp_pattern = re.compile("-(\\d+)\\.") - match = timestamp_pattern.search(filename) - return int(match.group(1)) if match else 0 diff --git a/dendrite/browser/sync_api/_remote_impl/browserbase/_impl.py b/dendrite/browser/sync_api/_remote_impl/browserbase/_impl.py deleted file mode 100644 index 3a94417..0000000 --- a/dendrite/browser/sync_api/_remote_impl/browserbase/_impl.py +++ /dev/null @@ -1,64 +0,0 @@ -from typing import TYPE_CHECKING, Optional -from dendrite.browser._common._exceptions.dendrite_exception import ( - BrowserNotLaunchedError, -) -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.remote.browserbase_config import BrowserbaseConfig - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -from loguru import logger -from playwright.sync_api import Playwright -from dendrite.browser.sync_api._remote_impl.browserbase._client import BrowserbaseClient -from dendrite.browser.sync_api._remote_impl.browserbase._download import ( - BrowserbaseDownload, -) - - -class BrowserBaseImpl(ImplBrowser): - - def __init__(self, settings: BrowserbaseConfig) -> None: - self.settings = settings - self._client = BrowserbaseClient( - self.settings.api_key, self.settings.project_id - ) - self._session_id: Optional[str] = None - - def stop_session(self): - if self._session_id: - self._client.stop_session(self._session_id) - - def start_browser(self, playwright: Playwright, pw_options: dict): - logger.debug("Starting browser") - self._session_id = self._client.create_session() - url = self._client.connect_url(self.settings.enable_proxy, self._session_id) - logger.debug(f"Connecting to browser at {url}") - return playwright.chromium.connect_over_cdp(url) - - def configure_context(self, browser: "Dendrite"): - logger.debug("Configuring browser context") - page = browser.get_active_page() - pw_page = page.playwright_page - if browser.browser_context is None: - raise BrowserNotLaunchedError() - client = browser.browser_context.new_cdp_session(pw_page) - client.send( - "Browser.setDownloadBehavior", - {"behavior": "allow", "downloadPath": "downloads", "eventsEnabled": True}, - ) - - def get_download( - self, - dendrite_browser: "Dendrite", - pw_page: PlaywrightPage, - timeout: float = 30000, - ) -> BrowserbaseDownload: - if not self._session_id: - raise ValueError( - "Downloads are not enabled for this provider. Specify enable_downloads=True in the constructor" - ) - logger.debug("Getting download") - download = dendrite_browser._download_handler.get_data(pw_page, timeout) - self._client.save_downloads_on_disk(self._session_id, download.path(), 30) - return BrowserbaseDownload(self._session_id, download, self._client) diff --git a/dendrite/browser/sync_api/_remote_impl/browserless/__init__.py b/dendrite/browser/sync_api/_remote_impl/browserless/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/_remote_impl/browserless/_impl.py b/dendrite/browser/sync_api/_remote_impl/browserless/_impl.py deleted file mode 100644 index d254015..0000000 --- a/dendrite/browser/sync_api/_remote_impl/browserless/_impl.py +++ /dev/null @@ -1,57 +0,0 @@ -import json -from typing import TYPE_CHECKING, Optional -from dendrite.browser._common._exceptions.dendrite_exception import ( - BrowserNotLaunchedError, -) -from dendrite.browser.sync_api._core._impl_browser import ImplBrowser -from dendrite.browser.sync_api._core._type_spec import PlaywrightPage -from dendrite.browser.remote.browserless_config import BrowserlessConfig - -if TYPE_CHECKING: - from dendrite.browser.sync_api._core.dendrite_browser import Dendrite -import urllib.parse -from loguru import logger -from playwright.sync_api import Playwright -from dendrite.browser.sync_api._remote_impl.browserbase._client import BrowserbaseClient -from dendrite.browser.sync_api._remote_impl.browserbase._download import ( - BrowserbaseDownload, -) - - -class BrowserlessImpl(ImplBrowser): - - def __init__(self, settings: BrowserlessConfig) -> None: - self.settings = settings - self._session_id: Optional[str] = None - - def stop_session(self): - pass - - def start_browser(self, playwright: Playwright, pw_options: dict): - logger.debug("Starting browser") - url = self._format_connection_url(pw_options) - logger.debug(f"Connecting to browser at {url}") - return playwright.chromium.connect_over_cdp(url) - - def _format_connection_url(self, pw_options: dict) -> str: - url = self.settings.url.rstrip("?").rstrip("/") - query = { - "token": self.settings.api_key, - "blockAds": self.settings.block_ads, - "launch": json.dumps(pw_options), - } - if self.settings.proxy: - query["proxy"] = (self.settings.proxy,) - query["proxyCountry"] = (self.settings.proxy_country,) - return f"{url}?{urllib.parse.urlencode(query)}" - - def configure_context(self, browser: "Dendrite"): - pass - - def get_download( - self, - dendrite_browser: "Dendrite", - pw_page: PlaywrightPage, - timeout: float = 30000, - ) -> BrowserbaseDownload: - raise NotImplementedError("Downloads are not supported for Browserless") diff --git a/dendrite/logic/config.py b/dendrite/logic/config.py index 4735e71..bbb4e45 100644 --- a/dendrite/logic/config.py +++ b/dendrite/logic/config.py @@ -9,10 +9,13 @@ class Config: def __init__( self, - cache_path: Optional[Union[str, Path]] = None, + root_path: Union[str, Path] = ".dendrite", + cache_path: Union[str, Path] = "cache", + auth_session_path: Union[str, Path] = "auth", llm_config: Optional[LLMConfig] = None, ): - self.cache_path = Path(cache_path) if cache_path else Path("./.dendrite/cache") + self.cache_path = root_path / Path(cache_path) self.llm_config = llm_config or LLMConfig() self.extract_cache = FileCache(Script, self.cache_path / "extract.json") self.element_cache = FileCache(Selector, self.cache_path / "get_element.json") + self.auth_session_path = root_path / Path(auth_session_path) From 3dcb11a2a2c36c5d17d93609c8bd609493aad88d Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Thu, 5 Dec 2024 10:48:00 +0100 Subject: [PATCH 11/18] add local storagestate storage --- dendrite/_cli/main.py | 15 +- .../browser/async_api/_core/_impl_browser.py | 26 +-- .../async_api/_core/_local_browser_impl.py | 67 +----- dendrite/browser/async_api/_core/_utils.py | 40 +--- .../async_api/_core/dendrite_browser.py | 197 ++++++++++-------- dendrite/logic/cache/file_cache.py | 33 ++- dendrite/logic/cache/storage_cache.py | 0 dendrite/logic/config.py | 5 + dendrite/logic/extract/extract_agent.py | 10 +- dendrite/logic/extract/scroll_agent.py | 2 + pyproject.toml | 1 + 11 files changed, 183 insertions(+), 213 deletions(-) create mode 100644 dendrite/logic/cache/storage_cache.py diff --git a/dendrite/_cli/main.py b/dendrite/_cli/main.py index e2ec376..38d973f 100644 --- a/dendrite/_cli/main.py +++ b/dendrite/_cli/main.py @@ -25,8 +25,7 @@ async def setup_auth(url: str, profile_name: str): async with AsyncDendrite() as browser: await browser.setup_auth( url=url, - profile_name=profile_name, - message="Please log in to the website. Once done, press Enter to continue..." + message="Please log in to the website. Once done, press Enter to continue...", ) print(f"Authentication profile '{profile_name}' has been saved successfully.") except Exception as e: @@ -36,11 +35,17 @@ async def setup_auth(url: str, profile_name: str): def main(): parser = argparse.ArgumentParser(description="Dendrite SDK CLI tool") - parser.add_argument("command", choices=["install", "auth"], help="Command to execute") - + parser.add_argument( + "command", choices=["install", "auth"], help="Command to execute" + ) + # Add auth-specific arguments parser.add_argument("--url", help="URL to navigate to for authentication") - parser.add_argument("--profile", default="default", help="Name for the authentication profile (default: 'default')") + parser.add_argument( + "--profile", + default="default", + help="Name for the authentication profile (default: 'default')", + ) args = parser.parse_args() diff --git a/dendrite/browser/async_api/_core/_impl_browser.py b/dendrite/browser/async_api/_core/_impl_browser.py index aafa93c..9d9aff3 100644 --- a/dendrite/browser/async_api/_core/_impl_browser.py +++ b/dendrite/browser/async_api/_core/_impl_browser.py @@ -5,7 +5,13 @@ if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from playwright.async_api import Browser, Download, Playwright, BrowserContext +from playwright.async_api import ( + Browser, + Download, + Playwright, + BrowserContext, + StorageState, +) from dendrite.browser.async_api._core._type_spec import PlaywrightPage @@ -30,35 +36,21 @@ async def get_download( Exception: If there is an issue retrieving the download event. """ - @overload - @abstractmethod - async def start_browser( - self, playwright: Playwright, pw_options: dict, user_data_dir: str - ) -> BrowserContext: ... - - @overload - @abstractmethod - async def start_browser( - self, playwright: Playwright, pw_options: dict, user_data_dir: None = None - ) -> Browser: ... - @abstractmethod async def start_browser( self, playwright: Playwright, pw_options: dict, - user_data_dir: Optional[str] = None, - ) -> Union[Browser, BrowserContext]: + ) -> Browser: """ Starts the browser session. Args: playwright: The playwright instance pw_options: Playwright launch options - user_data_dir: Optional path to Chrome user data directory for persistent context Returns: - Union[Browser, BrowserContext]: Either a Browser instance or BrowserContext for persistent sessions + Browser: A Browser instance """ @abstractmethod diff --git a/dendrite/browser/async_api/_core/_local_browser_impl.py b/dendrite/browser/async_api/_core/_local_browser_impl.py index aa5fd0a..009330c 100644 --- a/dendrite/browser/async_api/_core/_local_browser_impl.py +++ b/dendrite/browser/async_api/_core/_local_browser_impl.py @@ -8,7 +8,13 @@ if TYPE_CHECKING: from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite -from playwright.async_api import Browser, Download, Playwright, BrowserContext +from playwright.async_api import ( + Browser, + Download, + Playwright, + BrowserContext, + StorageState, +) from dendrite.browser.async_api._core._impl_browser import ImplBrowser from dendrite.browser.async_api._core._type_spec import PlaywrightPage @@ -22,67 +28,12 @@ class LocalImpl(ImplBrowser): def __init__(self) -> None: pass - @overload - async def start_browser( - self, playwright: Playwright, pw_options: dict, user_data_dir: str - ) -> BrowserContext: ... - - @overload - async def start_browser( - self, playwright: Playwright, pw_options: dict, user_data_dir: None = None - ) -> Browser: ... - async def start_browser( self, playwright: Playwright, pw_options: dict, - user_data_dir: Optional[str] = None, - ) -> Union[Browser, BrowserContext]: - if user_data_dir is not None: - args = { - "user_data_dir": user_data_dir, - "ignore_default_args": ["--enable-automation"], - } - - # Check if profile is locked - singleton_lock = os.path.join(user_data_dir, "SingletonLock") - logger.warning(f"Checking for singleton lock at {os.path.abspath(singleton_lock)}") - - res = Path("/Users/arian/Projects/dendrite/dendrite-python-sdk/browser_profiles/my_google_profile/SingletonLock").is_symlink() - if res: - is_locked = True - else: - is_locked = False - - logger.warning(f"Profile is locked: {is_locked}") - if is_locked: - logger.warning("Profile is locked, creating a temporary copy") - # Create a temporary copy of the user data directory - temp_dir = tempfile.mkdtemp() - temp_user_data = os.path.join(temp_dir, "chrome_data") - - def copy_ignore(src, names): - return [ - 'SingletonSocket', 'SingletonLock', 'SingletonCookie', - 'DeferredBrowserMetrics', 'RunningChromeVersion', - '.org.chromium.Chromium.*' - ] - - # Copy tree with error handling - shutil.copytree( - user_data_dir, - temp_user_data, - ignore=copy_ignore, - dirs_exist_ok=True - ) - args["user_data_dir"] = temp_user_data - - pw_options.update(args) - return await playwright.chromium.launch_persistent_context( - channel="chrome", - **pw_options - ) - + storage_state: Optional[StorageState] = None, + ) -> Browser: return await playwright.chromium.launch(**pw_options) async def get_download( diff --git a/dendrite/browser/async_api/_core/_utils.py b/dendrite/browser/async_api/_core/_utils.py index e9074f1..1da49cc 100644 --- a/dendrite/browser/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_core/_utils.py @@ -3,6 +3,7 @@ from bs4 import BeautifulSoup from loguru import logger from playwright.async_api import ElementHandle, Error, Frame, FrameLocator +import tldextract from dendrite.browser.async_api._core._type_spec import PlaywrightPage from dendrite.browser.async_api._core.dendrite_element import AsyncElement @@ -16,38 +17,13 @@ from dendrite.browser.async_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT from dendrite.logic.dom.strip import mild_strip_in_place -import os -import platform -from pathlib import Path - - -def get_chrome_user_data_dir() -> str: - """ - Get the default Chrome user data directory based on the operating system. - - Returns: - str: Path to Chrome user data directory - """ - system = platform.system() - home = Path.home() - - if system == "Windows": - return str(Path(os.getenv("LOCALAPPDATA", "")) / "Google/Chrome/User Data") - elif system == "Darwin": # macOS - return str(home / "Library/Application Support/Google/Chrome/") - elif system == "Linux": - return str(home / ".config/google-chrome") - else: - raise NotImplementedError(f"Unsupported operating system: {system}") - - -def chrome_profile_exists() -> bool: - """Check if a Chrome profile exists in the default location.""" - try: - user_data_dir = get_chrome_user_data_dir() - return Path(user_data_dir).exists() - except: - return False + +def get_domain_w_suffix(url: str) -> str: + parsed_url = tldextract.extract(url) + if parsed_url.suffix == "": + raise ValueError(f"Invalid URL: {url}") + + return f"{parsed_url.domain}.{parsed_url.suffix}" async def expand_iframes( diff --git a/dendrite/browser/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/_core/dendrite_browser.py index 88837ae..bdf826a 100644 --- a/dendrite/browser/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/_core/dendrite_browser.py @@ -4,19 +4,15 @@ from abc import ABC from typing import Any, List, Optional, Sequence, Union from uuid import uuid4 -import shutil -import tempfile -import uuid from loguru import logger from playwright.async_api import ( - BrowserContext, Download, Error, FileChooser, FilePayload, - Playwright, async_playwright, + StorageState, ) from dendrite.browser._common._exceptions.dendrite_exception import ( @@ -29,6 +25,7 @@ from dendrite.browser.async_api._core._impl_mapping import get_impl from dendrite.browser.async_api._core._managers.page_manager import PageManager from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api._core._utils import get_domain_w_suffix from dendrite.browser.async_api._core.dendrite_page import AsyncPage from dendrite.browser.async_api._core.event_sync import EventSync from dendrite.browser.async_api._core.mixin import ( @@ -45,7 +42,6 @@ from dendrite.browser.remote import Providers from dendrite.logic.config import Config from dendrite.logic.interfaces import AsyncProtocol -from ._utils import get_chrome_user_data_dir, chrome_profile_exists class AsyncDendrite( @@ -91,47 +87,30 @@ def __init__( }, remote_config: Optional[Providers] = None, config: Optional[Config] = None, - use_chrome_profile: bool = False, + auth: Optional[Union[List[str], str]] = None, ): """ - Initializes AsyncDendrite with API keys and Playwright options. + Initialize AsyncDendrite with optional domain authentication. Args: - dendrite_api_key (Optional[str]): The Dendrite API key. If not provided, it's fetched from the environment variables. - openai_api_key (Optional[str]): Your own OpenAI API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. - anthropic_api_key (Optional[str]): The own Anthropic API key, provide it, along with other custom API keys, if you wish to use Dendrite without paying for a license. - playwright_options (Any): Options for configuring Playwright. Defaults to running in non-headless mode with stealth arguments. - remote_config (Optional[Providers]): Remote browser provider configuration - config (Optional[Config]): Configuration object - use_chrome_profile (bool): Whether to try using the existing Chrome profile. Defaults to False. - - Raises: - MissingApiKeyError: If the Dendrite API key is not provided or found in the environment variables. + playwright_options: Options for configuring Playwright + remote_config: Remote browser provider configuration + config: Configuration object + auth: List of domains or single domain to load authentication state for """ - - # api_config = APIConfig( - # dendrite_api_key=dendrite_api_key or os.environ.get("DENDRITE_API_KEY"), - # openai_api_key=openai_api_key, - # anthropic_api_key=anthropic_api_key, - # ) - self._impl = self._get_impl(remote_config) - - # self.api_config = api_config - self.playwright: Optional[Playwright] = None - self.browser_context: Optional[BrowserContext] = None + self._playwright_options = playwright_options + self._config = config or Config() + auth_url = [auth] if isinstance(auth, str) else auth or [] + self._auth_domains = [get_domain_w_suffix(url) for url in auth_url] self._id = uuid4().hex - self._playwright_options = playwright_options self._active_page_manager: Optional[PageManager] = None self._user_id: Optional[str] = None self._upload_handler = EventSync(event_type=FileChooser) self._download_handler = EventSync(event_type=Download) self.closed = False - self._config = config or Config() self._browser_api_client: AsyncProtocol = AsyncProtocol(self._config) - self._use_chrome_profile = use_chrome_profile - self._temp_profile_dir = None @property def pages(self) -> List[AsyncPage]: @@ -291,20 +270,24 @@ async def _launch(self): os.environ["PW_TEST_SCREENSHOT_NO_FONTS_READY"] = "1" self._playwright = await async_playwright().start() - browser = None - # Create temporary copy of Chrome profile if requested - if self._use_chrome_profile and chrome_profile_exists(): - original_dir = get_chrome_user_data_dir() + # Get and merge storage states for authenticated domains + storage_states = [] + for domain in self._auth_domains: + state = await self._get_domain_storage_state(domain) - context_options = {**self._playwright_options} + if state: + storage_states.append(state) - self.browser_context = await self._impl.start_browser( - self._playwright, context_options, "browser_profiles/my_google_profile" - ) + # Launch browser + browser = await self._impl.start_browser( + self._playwright, self._playwright_options + ) + + # Create context with merged storage state if available + if storage_states: + merged_state = await self._merge_storage_states(storage_states) + self.browser_context = await browser.new_context(storage_state=merged_state) else: - browser = await self._impl.start_browser( - self._playwright, self._playwright_options - ) self.browser_context = ( browser.contexts[0] if len(browser.contexts) > 0 @@ -333,37 +316,36 @@ async def add_cookies(self, cookies): async def close(self): """ - Closes the browser and cleans up temporary profile directory if it exists. + Closes the browser and updates storage states for authenticated domains before cleanup. - This method stops the Playwright instance, closes the browser context + This method updates the storage states for authenticated domains, stops the Playwright + instance, and closes the browser context. Returns: None Raises: - Exception: If there is an issue closing the browser or uploading session data. + Exception: If there is an issue closing the browser or updating session data. """ - self.closed = True + try: - if self.browser_context: + if self.browser_context and self._auth_domains: + # Update storage state for each authenticated domain + for domain in self._auth_domains: + await self.save_auth(domain) + await self._impl.stop_session() await self.browser_context.close() except Error: pass + try: if self._playwright: await self._playwright.stop() except (AttributeError, Exception): pass - # Clean up temporary profile directory - if self._temp_profile_dir and os.path.exists(self._temp_profile_dir): - try: - shutil.rmtree(self._temp_profile_dir) - except Exception as e: - logger.warning(f"Failed to remove temporary profile directory: {e}") - def _is_launched(self): """ Checks whether the browser context has been launched. @@ -459,50 +441,69 @@ async def _get_filechooser( """ return await self._upload_handler.get_data(pw_page, timeout=timeout) + async def save_auth(self, url: str) -> None: + """ + Save authentication state for a specific domain. + + Args: + domain (str): Domain to save authentication for (e.g., "github.com") + """ + if not self.browser_context: + raise DendriteException("Browser context not initialized") + + domain = get_domain_w_suffix(url) + + # Get current storage state + storage_state = await self.browser_context.storage_state() + + # Filter storage state for specific domain + filtered_state = { + "origins": [ + origin + for origin in storage_state.get("origins", []) + if domain in origin.get("origin", "") + ], + "cookies": [ + cookie + for cookie in storage_state.get("cookies", []) + if domain in cookie.get("domain", "") + ], + } + + # Save to cache + self._config.storage_cache.set( + {"domain": domain}, StorageState(**filtered_state) + ) + async def setup_auth( self, url: str, message: str = "Please log in to the website. Once done, press Enter to continue...", - profile_name: str = "default", ) -> None: """ - Launches a browser session for user login and saves the profile data. + Set up authentication for a specific URL. Args: - profile_name (str): Name for the profile to be saved url (str): URL to navigate to for login - message (str): Custom message to show while waiting for user input - - Returns: - None + message (str): Message to show while waiting for user input """ - # Create profiles directory if it doesn't exist - profiles_dir = self._config.auth_session_path - profile_path = profiles_dir / profile_name - profiles_dir.mkdir(exist_ok=True) + # Extract domain from URL + # domain = urlparse(url).netloc + # if not domain: + # domain = urlparse(f"https://{url}").netloc - # Launch browser with temporary profile + domain = get_domain_w_suffix(url) try: # Start Playwright self._playwright = await async_playwright().start() - # Launch persistent context - context_options = { - "headless": False, # Always show browser for login - "args": STEALTH_ARGS, - } - - self.browser_context = ( - await self._playwright.chromium.launch_persistent_context( - user_data_dir=profile_path, - ignore_default_args=["--enable-automation"], - channel="chrome", - **context_options, - ) + # Launch browser with normal context + browser = await self._impl.start_browser( + self._playwright, {**self._playwright_options, "headless": False} ) - # Set up page manager + self.browser_context = await browser.new_context() self._active_page_manager = PageManager(self, self.browser_context) # Navigate to login page @@ -512,6 +513,38 @@ async def setup_auth( print(message) input() + # Save the storage state for this domain + await self.save_auth(domain) + finally: # Clean up await self.close() + + async def _get_domain_storage_state(self, domain: str) -> Optional[StorageState]: + """Get storage state for a specific domain""" + return self._config.storage_cache.get({"domain": domain}) + + async def _merge_storage_states(self, states: List[StorageState]) -> StorageState: + """Merge multiple storage states into one""" + merged = {"origins": [], "cookies": []} + seen_origins = set() + seen_cookies = set() + + for state in states: + # Merge origins + for origin in state.get("origins", []): + origin_key = origin.get("origin", "") + if origin_key not in seen_origins: + merged["origins"].append(origin) + seen_origins.add(origin_key) + + # Merge cookies + for cookie in state.get("cookies", []): + cookie_key = ( + f"{cookie.get('name')}:{cookie.get('domain')}:{cookie.get('path')}" + ) + if cookie_key not in seen_cookies: + merged["cookies"].append(cookie) + seen_cookies.add(cookie_key) + + return StorageState(**merged) diff --git a/dendrite/logic/cache/file_cache.py b/dendrite/logic/cache/file_cache.py index 12405d1..d5e165a 100644 --- a/dendrite/logic/cache/file_cache.py +++ b/dendrite/logic/cache/file_cache.py @@ -2,11 +2,11 @@ import threading from hashlib import md5 from pathlib import Path -from typing import Dict, Generic, Type, TypeVar, Union +from typing import Dict, Generic, Type, TypeVar, Union, Any, Mapping from pydantic import BaseModel -T = TypeVar("T", bound=BaseModel) +T = TypeVar("T", bound=Union[BaseModel, Mapping[Any, Any]]) class FileCache(Generic[T]): @@ -32,21 +32,32 @@ def _load_cache(self) -> None: json_string = self.filepath.read_text() raw_dict = json.loads(json_string) - # Convert each entry back to the model class - self.cache = { - k: self.model_class.model_validate_json(json.dumps(v)) - for k, v in raw_dict.items() - } + # Convert each entry based on model_class type + self.cache = {} + for k, v in raw_dict.items(): + if issubclass(self.model_class, BaseModel): + self.cache[k] = self.model_class.model_validate_json( + json.dumps(v) + ) + else: + # For any Mapping type (dict, TypedDict, etc) + self.cache[k] = v except (json.JSONDecodeError, FileNotFoundError): self.cache = {} def _save_cache(self, cache_dict: Dict[str, T]) -> None: """Save cache to file""" with self.lock: - # Convert models to dict before saving - serializable_dict = { - k: json.loads(v.model_dump_json()) for k, v in cache_dict.items() - } + # Convert entries based on their type + serializable_dict = {} + for k, v in cache_dict.items(): + if isinstance(v, BaseModel): + serializable_dict[k] = json.loads(v.model_dump_json()) + elif isinstance(v, Mapping): + serializable_dict[k] = dict(v) # Convert any Mapping to dict + else: + raise ValueError(f"Unsupported type for cache value: {type(v)}") + self.filepath.write_text(json.dumps(serializable_dict, indent=2)) def get(self, key: Union[str, Dict[str, str]]) -> Union[T, None]: diff --git a/dendrite/logic/cache/storage_cache.py b/dendrite/logic/cache/storage_cache.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/config.py b/dendrite/logic/config.py index bbb4e45..6295b93 100644 --- a/dendrite/logic/config.py +++ b/dendrite/logic/config.py @@ -5,6 +5,8 @@ from dendrite.models.scripts import Script from dendrite.models.selector import Selector +from playwright.async_api import StorageState + class Config: def __init__( @@ -18,4 +20,7 @@ def __init__( self.llm_config = llm_config or LLMConfig() self.extract_cache = FileCache(Script, self.cache_path / "extract.json") self.element_cache = FileCache(Selector, self.cache_path / "get_element.json") + self.storage_cache = FileCache( + StorageState, self.cache_path / "storage_state.json" + ) self.auth_session_path = root_path / Path(auth_session_path) diff --git a/dendrite/logic/extract/extract_agent.py b/dendrite/logic/extract/extract_agent.py index 03e894f..3cfce36 100644 --- a/dendrite/logic/extract/extract_agent.py +++ b/dendrite/logic/extract/extract_agent.py @@ -3,7 +3,7 @@ import sys from typing import List, Union -from loguru import logger +from dendrite import logger from dendrite.logic.cache.utils import save_script from dendrite.logic.config import Config @@ -107,13 +107,7 @@ async def code_script_from_found_expanded_html_tags( self, extract_page_dto: ExtractDTO, expanded_html ): - agent_logger = logger.bind( - scope="extract", step="generate_code" - ) # agent_logger.info("Starting code_script_from_found_expanded_html_tags method") - agent_logger.remove() - fmt = "{time: HH:mm:ss.SSS} | {level: <8} | {message}" - agent_logger.add(sys.stderr, level="DEBUG", format=fmt) - messages = [] + agent_logger = logger.bind(scope="extract", step="generate_code") user_prompt = create_script_prompt_segmented_html( extract_page_dto.combined_prompt, diff --git a/dendrite/logic/extract/scroll_agent.py b/dendrite/logic/extract/scroll_agent.py index ea18145..7c72225 100644 --- a/dendrite/logic/extract/scroll_agent.py +++ b/dendrite/logic/extract/scroll_agent.py @@ -73,6 +73,8 @@ def __init__(self, page_information: PageInformation, llm_config: LLMConfig): ErrorRes(), ] + self.logger = logger.bind(agent="scroll_agent") + async def scroll_through_page( self, combined_prompt: str, diff --git a/pyproject.toml b/pyproject.toml index 897d9f5..70f9847 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ markdownify = "^0.13.1" litellm = "^1.52.9" pillow = "^11.0.0" json-repair = "^0.30.1" +tldextract = "^5.1.3" [tool.poetry.group.dev.dependencies] From dba1a18bd7d6e0edf58aa0a658942f5bc2ede4c9 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Thu, 5 Dec 2024 14:57:37 +0100 Subject: [PATCH 12/18] refactor to flatten project structure --- dendrite/__init__.py | 22 +- .../async_api/_core => _cli}/__init__.py | 0 dendrite/_cli/main.py | 3 +- .../_core/_managers => }/__init__.py | 0 dendrite/browser/async_api/__init__.py | 8 +- .../browser/async_api/_core/_type_spec.py | 44 -- .../async_api/_core/models/response.py | 55 -- .../{_core/event_sync.py => _event_sync.py} | 0 .../browser/async_api/{_core => }/_utils.py | 49 +- .../__init__.py | 0 .../browserbase/__init__.py | 0 .../browserbase/_client.py | 0 .../browserbase/_download.py | 4 +- .../browserbase/_impl.py | 16 +- .../browserless/__init__.py | 0 .../browserless/_impl.py | 12 +- .../impl_mapping.py} | 16 +- .../local/_impl.py} | 19 +- .../async_api/{_core => }/dendrite_browser.py | 31 +- .../async_api/{_core => }/dendrite_element.py | 14 +- .../async_api/{_core => }/dendrite_page.py | 39 +- .../async_api/{_core/_js => js}/__init__.py | 0 .../{_core/_js => js}/eventListenerPatch.js | 0 .../{_core/_js => js}/generateDendriteIDs.js | 0 .../_js => js}/generateDendriteIDsIframe.js | 0 .../{_core/models => manager}/__init__.py | 0 .../navigation_tracker.py | 2 +- .../_managers => manager}/page_manager.py | 6 +- .../screenshot_manager.py | 2 +- .../async_api/{_core => }/mixin/__init__.py | 1 - .../async_api/{_core => }/mixin/ask.py | 12 +- .../async_api/{_core => }/mixin/click.py | 5 +- .../async_api/{_core => }/mixin/extract.py | 16 +- .../{_core => }/mixin/fill_fields.py | 5 +- .../{_core => }/mixin/get_element.py | 103 +--- .../async_api/{_core => }/mixin/keyboard.py | 3 +- .../async_api/{_core => }/mixin/markdown.py | 4 +- .../async_api/{_core => }/mixin/screenshot.py | 2 +- .../async_api/{_core => }/mixin/wait_for.py | 5 +- .../async_api/{_dom => protocol}/__init__.py | 0 .../browser_protocol.py} | 33 +- .../download_protocol.py} | 0 .../{_core => }/protocol/page_protocol.py | 8 +- dendrite/browser/async_api/types.py | 15 + dendrite/browser/sync_api/__init__.py | 6 + dendrite/browser/sync_api/_event_sync.py | 45 ++ dendrite/browser/sync_api/_utils.py | 124 +++++ .../browser/sync_api/browser_impl/__init__.py | 3 + .../browser_impl/browserbase/__init__.py | 3 + .../browser_impl/browserbase/_client.py | 63 +++ .../browser_impl/browserbase/_download.py | 53 ++ .../browser_impl/browserbase/_impl.py | 62 +++ .../browser_impl/browserless/__init__.py | 0 .../browser_impl/browserless/_impl.py | 57 ++ .../sync_api/browser_impl/impl_mapping.py | 29 ++ .../sync_api/browser_impl/local/__init__.py | 0 .../sync_api/browser_impl/local/_impl.py | 45 ++ dendrite/browser/sync_api/dendrite_browser.py | 485 ++++++++++++++++++ dendrite/browser/sync_api/dendrite_element.py | 237 +++++++++ dendrite/browser/sync_api/dendrite_page.py | 377 ++++++++++++++ dendrite/browser/sync_api/js/__init__.py | 11 + .../browser/sync_api/js/eventListenerPatch.js | 90 ++++ .../sync_api/js/generateDendriteIDs.js | 88 ++++ .../sync_api/js/generateDendriteIDsIframe.js | 93 ++++ dendrite/browser/sync_api/manager/__init__.py | 0 .../sync_api/manager/navigation_tracker.py | 67 +++ .../browser/sync_api/manager/page_manager.py | 82 +++ .../sync_api/manager/screenshot_manager.py | 50 ++ dendrite/browser/sync_api/mixin/__init__.py | 21 + dendrite/browser/sync_api/mixin/ask.py | 184 +++++++ dendrite/browser/sync_api/mixin/click.py | 56 ++ dendrite/browser/sync_api/mixin/extract.py | 277 ++++++++++ .../browser/sync_api/mixin/fill_fields.py | 76 +++ .../browser/sync_api/mixin/get_element.py | 250 +++++++++ dendrite/browser/sync_api/mixin/keyboard.py | 62 +++ dendrite/browser/sync_api/mixin/markdown.py | 23 + dendrite/browser/sync_api/mixin/screenshot.py | 20 + dendrite/browser/sync_api/mixin/wait_for.py | 53 ++ .../browser/sync_api/protocol/__init__.py | 0 .../sync_api/protocol/browser_protocol.py | 61 +++ .../sync_api/protocol/download_protocol.py | 20 + .../sync_api/protocol/page_protocol.py | 21 + dendrite/browser/sync_api/types.py | 12 + dendrite/logic/__init__.py | 4 + dendrite/logic/ask/__init__.py | 0 dendrite/logic/ask/ask.py | 2 - .../async_api.py => async_logic_engine.py} | 24 +- dendrite/logic/cache/__init__.py | 0 dendrite/logic/cache/element_cache.py | 3 +- dendrite/logic/cache/file_cache.py | 2 +- dendrite/logic/code/__init__.py | 0 dendrite/logic/config.py | 5 +- dendrite/logic/dom/__init__.py | 0 dendrite/logic/extract/__init__.py | 0 dendrite/logic/extract/extract_agent.py | 13 +- dendrite/logic/extract/scroll_agent.py | 4 +- dendrite/logic/factory.py | 15 - dendrite/logic/get_element/__init__.py | 0 .../logic/get_element/agents/segment_agent.py | 2 + dendrite/logic/get_element/get_element.py | 1 + dendrite/logic/interfaces/__init__.py | 8 - dendrite/logic/interfaces/cache.py | 5 - dendrite/logic/llm/__init__.py | 0 .../sync_api.py => sync_logic_engine.py} | 26 +- dendrite/logic/verify_interaction/__init__.py | 0 .../verify_interaction.py | 0 dendrite/models/__init__.py | 0 dendrite/models/dto/__init__.py | 0 dendrite/models/response/__init__.py | 0 dendrite/models/response/ask_page_response.py | 2 +- dendrite/models/status.py | 1 - dendrite/remote/__init__.py | 6 + poetry.lock | 37 +- scripts/generate_sync.py | 2 +- tests/tests_async/conftest.py | 17 +- tests/tests_async/test_download.py | 4 +- tests/tests_async/tests.py | 1 + tests/tests_sync/conftest.py | 6 +- tests/tests_sync/test_context.py | 6 +- tests/tests_sync/test_download.py | 3 +- 120 files changed, 3483 insertions(+), 476 deletions(-) rename dendrite/{browser/async_api/_core => _cli}/__init__.py (100%) rename dendrite/browser/{async_api/_core/_managers => }/__init__.py (100%) delete mode 100644 dendrite/browser/async_api/_core/_type_spec.py delete mode 100644 dendrite/browser/async_api/_core/models/response.py rename dendrite/browser/async_api/{_core/event_sync.py => _event_sync.py} (100%) rename dendrite/browser/async_api/{_core => }/_utils.py (70%) rename dendrite/browser/async_api/{_remote_impl => browser_impl}/__init__.py (100%) rename dendrite/browser/async_api/{_remote_impl => browser_impl}/browserbase/__init__.py (100%) rename dendrite/browser/async_api/{_remote_impl => browser_impl}/browserbase/_client.py (100%) rename dendrite/browser/async_api/{_remote_impl => browser_impl}/browserbase/_download.py (93%) rename dendrite/browser/async_api/{_remote_impl => browser_impl}/browserbase/_impl.py (83%) rename dendrite/browser/async_api/{_remote_impl => browser_impl}/browserless/__init__.py (100%) rename dendrite/browser/async_api/{_remote_impl => browser_impl}/browserless/_impl.py (81%) rename dendrite/browser/async_api/{_core/_impl_mapping.py => browser_impl/impl_mapping.py} (60%) rename dendrite/browser/async_api/{_core/_local_browser_impl.py => browser_impl/local/_impl.py} (80%) rename dendrite/browser/async_api/{_core => }/dendrite_browser.py (96%) rename dendrite/browser/async_api/{_core => }/dendrite_element.py (96%) rename dendrite/browser/async_api/{_core => }/dendrite_page.py (91%) rename dendrite/browser/async_api/{_core/_js => js}/__init__.py (100%) rename dendrite/browser/async_api/{_core/_js => js}/eventListenerPatch.js (100%) rename dendrite/browser/async_api/{_core/_js => js}/generateDendriteIDs.js (100%) rename dendrite/browser/async_api/{_core/_js => js}/generateDendriteIDsIframe.js (100%) rename dendrite/browser/async_api/{_core/models => manager}/__init__.py (100%) rename dendrite/browser/async_api/{_core/_managers => manager}/navigation_tracker.py (97%) rename dendrite/browser/async_api/{_core/_managers => manager}/page_manager.py (94%) rename dendrite/browser/async_api/{_core/_managers => manager}/screenshot_manager.py (96%) rename dendrite/browser/async_api/{_core => }/mixin/__init__.py (99%) rename dendrite/browser/async_api/{_core => }/mixin/ask.py (96%) rename dendrite/browser/async_api/{_core => }/mixin/click.py (92%) rename dendrite/browser/async_api/{_core => }/mixin/extract.py (96%) rename dendrite/browser/async_api/{_core => }/mixin/fill_fields.py (95%) rename dendrite/browser/async_api/{_core => }/mixin/get_element.py (68%) rename dendrite/browser/async_api/{_core => }/mixin/keyboard.py (95%) rename dendrite/browser/async_api/{_core => }/mixin/markdown.py (89%) rename dendrite/browser/async_api/{_core => }/mixin/screenshot.py (87%) rename dendrite/browser/async_api/{_core => }/mixin/wait_for.py (94%) rename dendrite/browser/async_api/{_dom => protocol}/__init__.py (100%) rename dendrite/browser/async_api/{_core/_impl_browser.py => protocol/browser_protocol.py} (69%) rename dendrite/browser/async_api/{_core/models/download_interface.py => protocol/download_protocol.py} (100%) rename dendrite/browser/async_api/{_core => }/protocol/page_protocol.py (58%) create mode 100644 dendrite/browser/async_api/types.py create mode 100644 dendrite/browser/sync_api/__init__.py create mode 100644 dendrite/browser/sync_api/_event_sync.py create mode 100644 dendrite/browser/sync_api/_utils.py create mode 100644 dendrite/browser/sync_api/browser_impl/__init__.py create mode 100644 dendrite/browser/sync_api/browser_impl/browserbase/__init__.py create mode 100644 dendrite/browser/sync_api/browser_impl/browserbase/_client.py create mode 100644 dendrite/browser/sync_api/browser_impl/browserbase/_download.py create mode 100644 dendrite/browser/sync_api/browser_impl/browserbase/_impl.py create mode 100644 dendrite/browser/sync_api/browser_impl/browserless/__init__.py create mode 100644 dendrite/browser/sync_api/browser_impl/browserless/_impl.py create mode 100644 dendrite/browser/sync_api/browser_impl/impl_mapping.py create mode 100644 dendrite/browser/sync_api/browser_impl/local/__init__.py create mode 100644 dendrite/browser/sync_api/browser_impl/local/_impl.py create mode 100644 dendrite/browser/sync_api/dendrite_browser.py create mode 100644 dendrite/browser/sync_api/dendrite_element.py create mode 100644 dendrite/browser/sync_api/dendrite_page.py create mode 100644 dendrite/browser/sync_api/js/__init__.py create mode 100644 dendrite/browser/sync_api/js/eventListenerPatch.js create mode 100644 dendrite/browser/sync_api/js/generateDendriteIDs.js create mode 100644 dendrite/browser/sync_api/js/generateDendriteIDsIframe.js create mode 100644 dendrite/browser/sync_api/manager/__init__.py create mode 100644 dendrite/browser/sync_api/manager/navigation_tracker.py create mode 100644 dendrite/browser/sync_api/manager/page_manager.py create mode 100644 dendrite/browser/sync_api/manager/screenshot_manager.py create mode 100644 dendrite/browser/sync_api/mixin/__init__.py create mode 100644 dendrite/browser/sync_api/mixin/ask.py create mode 100644 dendrite/browser/sync_api/mixin/click.py create mode 100644 dendrite/browser/sync_api/mixin/extract.py create mode 100644 dendrite/browser/sync_api/mixin/fill_fields.py create mode 100644 dendrite/browser/sync_api/mixin/get_element.py create mode 100644 dendrite/browser/sync_api/mixin/keyboard.py create mode 100644 dendrite/browser/sync_api/mixin/markdown.py create mode 100644 dendrite/browser/sync_api/mixin/screenshot.py create mode 100644 dendrite/browser/sync_api/mixin/wait_for.py create mode 100644 dendrite/browser/sync_api/protocol/__init__.py create mode 100644 dendrite/browser/sync_api/protocol/browser_protocol.py create mode 100644 dendrite/browser/sync_api/protocol/download_protocol.py create mode 100644 dendrite/browser/sync_api/protocol/page_protocol.py create mode 100644 dendrite/browser/sync_api/types.py create mode 100644 dendrite/logic/__init__.py create mode 100644 dendrite/logic/ask/__init__.py rename dendrite/logic/{interfaces/async_api.py => async_logic_engine.py} (81%) create mode 100644 dendrite/logic/cache/__init__.py create mode 100644 dendrite/logic/code/__init__.py create mode 100644 dendrite/logic/dom/__init__.py create mode 100644 dendrite/logic/extract/__init__.py delete mode 100644 dendrite/logic/factory.py create mode 100644 dendrite/logic/get_element/__init__.py delete mode 100644 dendrite/logic/interfaces/__init__.py delete mode 100644 dendrite/logic/interfaces/cache.py create mode 100644 dendrite/logic/llm/__init__.py rename dendrite/logic/{interfaces/sync_api.py => sync_logic_engine.py} (85%) create mode 100644 dendrite/logic/verify_interaction/__init__.py rename dendrite/logic/{ => verify_interaction}/verify_interaction.py (100%) create mode 100644 dendrite/models/__init__.py create mode 100644 dendrite/models/dto/__init__.py create mode 100644 dendrite/models/response/__init__.py create mode 100644 dendrite/remote/__init__.py diff --git a/dendrite/__init__.py b/dendrite/__init__.py index b0777ff..8184634 100644 --- a/dendrite/__init__.py +++ b/dendrite/__init__.py @@ -1,36 +1,22 @@ import sys -from loguru import logger -from dendrite.browser.async_api import ( - AsyncDendrite, - AsyncElement, - AsyncPage, - AsyncElementsResponse, -) + +from dendrite._loggers.d_logger import logger +from dendrite.browser.async_api import AsyncDendrite, AsyncElement, AsyncPage +from dendrite.logic.config import Config from dendrite.browser.sync_api import ( Dendrite, Element, Page, - ElementsResponse, ) -from dendrite.logic.config import Config - -# logger.remove() - -# fmt = "{time: HH:mm:ss.SSS} | {level: <8}- {message}" - -# logger.add(sys.stderr, level="INFO", format=fmt) - __all__ = [ "AsyncDendrite", "AsyncElement", "AsyncPage", - "AsyncElementsResponse", "Dendrite", "Element", "Page", - "ElementsResponse", "Config", ] diff --git a/dendrite/browser/async_api/_core/__init__.py b/dendrite/_cli/__init__.py similarity index 100% rename from dendrite/browser/async_api/_core/__init__.py rename to dendrite/_cli/__init__.py diff --git a/dendrite/_cli/main.py b/dendrite/_cli/main.py index 38d973f..8f86ac5 100644 --- a/dendrite/_cli/main.py +++ b/dendrite/_cli/main.py @@ -1,7 +1,8 @@ import argparse +import asyncio import subprocess import sys -import asyncio + from dendrite.browser.async_api import AsyncDendrite from dendrite.logic.config import Config diff --git a/dendrite/browser/async_api/_core/_managers/__init__.py b/dendrite/browser/__init__.py similarity index 100% rename from dendrite/browser/async_api/_core/_managers/__init__.py rename to dendrite/browser/__init__.py diff --git a/dendrite/browser/async_api/__init__.py b/dendrite/browser/async_api/__init__.py index eb4bd3c..87168f2 100644 --- a/dendrite/browser/async_api/__init__.py +++ b/dendrite/browser/async_api/__init__.py @@ -1,13 +1,11 @@ from loguru import logger -from ._core.dendrite_browser import AsyncDendrite -from ._core.dendrite_element import AsyncElement -from ._core.dendrite_page import AsyncPage -from ._core.models.response import AsyncElementsResponse +from .dendrite_browser import AsyncDendrite +from .dendrite_element import AsyncElement +from .dendrite_page import AsyncPage __all__ = [ "AsyncDendrite", "AsyncElement", "AsyncPage", - "AsyncElementsResponse", ] diff --git a/dendrite/browser/async_api/_core/_type_spec.py b/dendrite/browser/async_api/_core/_type_spec.py deleted file mode 100644 index 872e1c4..0000000 --- a/dendrite/browser/async_api/_core/_type_spec.py +++ /dev/null @@ -1,44 +0,0 @@ -import inspect -from typing import Any, Dict, Literal, Type, TypeVar, Union - -from playwright.async_api import Page -from pydantic import BaseModel - -Interaction = Literal["click", "fill", "hover"] - -T = TypeVar("T") -PydanticModel = TypeVar("PydanticModel", bound=BaseModel) -PrimitiveTypes = PrimitiveTypes = Union[Type[bool], Type[int], Type[float], Type[str]] -JsonSchema = Dict[str, Any] -TypeSpec = Union[PrimitiveTypes, PydanticModel, JsonSchema] - -PlaywrightPage = Page - - -def to_json_schema(type_spec: TypeSpec) -> Dict[str, Any]: - if isinstance(type_spec, dict): - # Assume it's already a JSON schema - return type_spec - if inspect.isclass(type_spec) and issubclass(type_spec, BaseModel): - # Convert Pydantic model to JSON schema - return type_spec.model_json_schema() - if type_spec in (bool, int, float, str): - # Convert basic Python types to JSON schema - type_map = {bool: "boolean", int: "integer", float: "number", str: "string"} - return {"type": type_map[type_spec]} - - raise ValueError(f"Unsupported type specification: {type_spec}") - - -def convert_to_type_spec(type_spec: TypeSpec, return_data: Any) -> TypeSpec: - if isinstance(type_spec, type): - if issubclass(type_spec, BaseModel): - return type_spec.model_validate(return_data) - if type_spec in (str, float, bool, int): - return type_spec(return_data) - - raise ValueError(f"Unsupported type: {type_spec}") - if isinstance(type_spec, dict): - return return_data - - raise ValueError(f"Unsupported type specification: {type_spec}") diff --git a/dendrite/browser/async_api/_core/models/response.py b/dendrite/browser/async_api/_core/models/response.py deleted file mode 100644 index ba2c917..0000000 --- a/dendrite/browser/async_api/_core/models/response.py +++ /dev/null @@ -1,55 +0,0 @@ -from typing import Dict, Iterator - -from dendrite.browser.async_api._core.dendrite_element import AsyncElement - - -class AsyncElementsResponse: - """ - AsyncElementsResponse is a class that encapsulates a dictionary of Dendrite elements, - allowing for attribute-style access and other convenient interactions. - - This class is used to store and access the elements retrieved by the `get_elements` function. - The attributes of this class dynamically match the keys of the dictionary passed to the `get_elements` function, - allowing for direct attribute-style access to the corresponding `AsyncElement` objects. - - Attributes: - _data (Dict[str, AsyncElement]): A dictionary where keys are the names of elements and values are the corresponding `AsyncElement` objects. - - Args: - data (Dict[str, AsyncElement]): The dictionary of elements to be encapsulated by the class. - - Methods: - __getattr__(name: str) -> AsyncElement: - Allows attribute-style access to the elements in the dictionary. - - __getitem__(key: str) -> AsyncElement: - Enables dictionary-style access to the elements. - - __iter__() -> Iterator[str]: - Provides an iterator over the keys in the dictionary. - - __repr__() -> str: - Returns a string representation of the class instance. - """ - - _data: Dict[str, AsyncElement] - - def __init__(self, data: Dict[str, AsyncElement]): - self._data = data - - def __getattr__(self, name: str) -> AsyncElement: - try: - return self._data[name] - except KeyError: - raise AttributeError( - f"'{self.__class__.__name__}' object has no attribute '{name}'" - ) - - def __getitem__(self, key: str) -> AsyncElement: - return self._data[key] - - def __iter__(self) -> Iterator[str]: - return iter(self._data) - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self._data})" diff --git a/dendrite/browser/async_api/_core/event_sync.py b/dendrite/browser/async_api/_event_sync.py similarity index 100% rename from dendrite/browser/async_api/_core/event_sync.py rename to dendrite/browser/async_api/_event_sync.py diff --git a/dendrite/browser/async_api/_core/_utils.py b/dendrite/browser/async_api/_utils.py similarity index 70% rename from dendrite/browser/async_api/_core/_utils.py rename to dendrite/browser/async_api/_utils.py index 1da49cc..6bb14c8 100644 --- a/dendrite/browser/async_api/_core/_utils.py +++ b/dendrite/browser/async_api/_utils.py @@ -1,22 +1,24 @@ -from typing import TYPE_CHECKING, List, Optional, Union +import inspect +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +import tldextract from bs4 import BeautifulSoup from loguru import logger -from playwright.async_api import ElementHandle, Error, Frame, FrameLocator -import tldextract +from playwright.async_api import Error, Frame +from pydantic import BaseModel -from dendrite.browser.async_api._core._type_spec import PlaywrightPage -from dendrite.browser.async_api._core.dendrite_element import AsyncElement -from dendrite.browser.async_api._core.models.response import AsyncElementsResponse -from dendrite.models.response.get_element_response import GetElementResponse from dendrite.models.selector import Selector +from .dendrite_element import AsyncElement +from .types import PlaywrightPage, TypeSpec + if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_page import AsyncPage + from .dendrite_page import AsyncPage -from dendrite.browser.async_api._core._js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT from dendrite.logic.dom.strip import mild_strip_in_place +from .js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT + def get_domain_w_suffix(url: str) -> str: parsed_url = tldextract.extract(url) @@ -125,3 +127,32 @@ async def get_elements_from_selectors_soup( return dendrite_elements[0] if only_one else dendrite_elements return None + + +def to_json_schema(type_spec: TypeSpec) -> Dict[str, Any]: + if isinstance(type_spec, dict): + # Assume it's already a JSON schema + return type_spec + if inspect.isclass(type_spec) and issubclass(type_spec, BaseModel): + # Convert Pydantic model to JSON schema + return type_spec.model_json_schema() + if type_spec in (bool, int, float, str): + # Convert basic Python types to JSON schema + type_map = {bool: "boolean", int: "integer", float: "number", str: "string"} + return {"type": type_map[type_spec]} + + raise ValueError(f"Unsupported type specification: {type_spec}") + + +def convert_to_type_spec(type_spec: TypeSpec, return_data: Any) -> TypeSpec: + if isinstance(type_spec, type): + if issubclass(type_spec, BaseModel): + return type_spec.model_validate(return_data) + if type_spec in (str, float, bool, int): + return type_spec(return_data) + + raise ValueError(f"Unsupported type: {type_spec}") + if isinstance(type_spec, dict): + return return_data + + raise ValueError(f"Unsupported type specification: {type_spec}") diff --git a/dendrite/browser/async_api/_remote_impl/__init__.py b/dendrite/browser/async_api/browser_impl/__init__.py similarity index 100% rename from dendrite/browser/async_api/_remote_impl/__init__.py rename to dendrite/browser/async_api/browser_impl/__init__.py diff --git a/dendrite/browser/async_api/_remote_impl/browserbase/__init__.py b/dendrite/browser/async_api/browser_impl/browserbase/__init__.py similarity index 100% rename from dendrite/browser/async_api/_remote_impl/browserbase/__init__.py rename to dendrite/browser/async_api/browser_impl/browserbase/__init__.py diff --git a/dendrite/browser/async_api/_remote_impl/browserbase/_client.py b/dendrite/browser/async_api/browser_impl/browserbase/_client.py similarity index 100% rename from dendrite/browser/async_api/_remote_impl/browserbase/_client.py rename to dendrite/browser/async_api/browser_impl/browserbase/_client.py diff --git a/dendrite/browser/async_api/_remote_impl/browserbase/_download.py b/dendrite/browser/async_api/browser_impl/browserbase/_download.py similarity index 93% rename from dendrite/browser/async_api/_remote_impl/browserbase/_download.py rename to dendrite/browser/async_api/browser_impl/browserbase/_download.py index 354e9b3..7a92880 100644 --- a/dendrite/browser/async_api/_remote_impl/browserbase/_download.py +++ b/dendrite/browser/async_api/browser_impl/browserbase/_download.py @@ -7,10 +7,10 @@ from loguru import logger from playwright.async_api import Download -from dendrite.browser.async_api._core.models.download_interface import DownloadInterface -from dendrite.browser.async_api._remote_impl.browserbase._client import ( +from dendrite.browser.async_api.browser_impl.browserbase._client import ( BrowserbaseClient, ) +from dendrite.browser.async_api.protocol.download_protocol import DownloadInterface class AsyncBrowserbaseDownload(DownloadInterface): diff --git a/dendrite/browser/async_api/_remote_impl/browserbase/_impl.py b/dendrite/browser/async_api/browser_impl/browserbase/_impl.py similarity index 83% rename from dendrite/browser/async_api/_remote_impl/browserbase/_impl.py rename to dendrite/browser/async_api/browser_impl/browserbase/_impl.py index 0d1b6ee..b44b219 100644 --- a/dendrite/browser/async_api/_remote_impl/browserbase/_impl.py +++ b/dendrite/browser/async_api/browser_impl/browserbase/_impl.py @@ -3,25 +3,21 @@ from dendrite.browser._common._exceptions.dendrite_exception import ( BrowserNotLaunchedError, ) -from dendrite.browser.async_api._core._impl_browser import ImplBrowser -from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api.protocol.browser_protocol import BrowserProtocol +from dendrite.browser.async_api.types import PlaywrightPage from dendrite.browser.remote.browserbase_config import BrowserbaseConfig if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from dendrite.browser.async_api.dendrite_browser import AsyncDendrite from loguru import logger from playwright.async_api import Playwright -from dendrite.browser.async_api._remote_impl.browserbase._client import ( - BrowserbaseClient, -) -from dendrite.browser.async_api._remote_impl.browserbase._download import ( - AsyncBrowserbaseDownload, -) +from ._client import BrowserbaseClient +from ._download import AsyncBrowserbaseDownload -class BrowserBaseImpl(ImplBrowser): +class BrowserbaseImpl(BrowserProtocol): def __init__(self, settings: BrowserbaseConfig) -> None: self.settings = settings self._client = BrowserbaseClient( diff --git a/dendrite/browser/async_api/_remote_impl/browserless/__init__.py b/dendrite/browser/async_api/browser_impl/browserless/__init__.py similarity index 100% rename from dendrite/browser/async_api/_remote_impl/browserless/__init__.py rename to dendrite/browser/async_api/browser_impl/browserless/__init__.py diff --git a/dendrite/browser/async_api/_remote_impl/browserless/_impl.py b/dendrite/browser/async_api/browser_impl/browserless/_impl.py similarity index 81% rename from dendrite/browser/async_api/_remote_impl/browserless/_impl.py rename to dendrite/browser/async_api/browser_impl/browserless/_impl.py index 61c8a42..698557d 100644 --- a/dendrite/browser/async_api/_remote_impl/browserless/_impl.py +++ b/dendrite/browser/async_api/browser_impl/browserless/_impl.py @@ -4,27 +4,27 @@ from dendrite.browser._common._exceptions.dendrite_exception import ( BrowserNotLaunchedError, ) -from dendrite.browser.async_api._core._impl_browser import ImplBrowser -from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from dendrite.browser.async_api.protocol.browser_protocol import BrowserProtocol +from dendrite.browser.async_api.types import PlaywrightPage from dendrite.browser.remote.browserless_config import BrowserlessConfig if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from dendrite.browser.async_api.dendrite_browser import AsyncDendrite import urllib.parse from loguru import logger from playwright.async_api import Playwright -from dendrite.browser.async_api._remote_impl.browserbase._client import ( +from dendrite.browser.async_api.browser_impl.browserbase._client import ( BrowserbaseClient, ) -from dendrite.browser.async_api._remote_impl.browserbase._download import ( +from dendrite.browser.async_api.browser_impl.browserbase._download import ( AsyncBrowserbaseDownload, ) -class BrowserlessImpl(ImplBrowser): +class BrowserlessImpl(BrowserProtocol): def __init__(self, settings: BrowserlessConfig) -> None: self.settings = settings self._session_id: Optional[str] = None diff --git a/dendrite/browser/async_api/_core/_impl_mapping.py b/dendrite/browser/async_api/browser_impl/impl_mapping.py similarity index 60% rename from dendrite/browser/async_api/_core/_impl_mapping.py rename to dendrite/browser/async_api/browser_impl/impl_mapping.py index 60bccfa..d588769 100644 --- a/dendrite/browser/async_api/_core/_impl_mapping.py +++ b/dendrite/browser/async_api/browser_impl/impl_mapping.py @@ -1,17 +1,17 @@ from typing import Dict, Optional, Type -from dendrite.browser.async_api._core._impl_browser import ImplBrowser -from dendrite.browser.async_api._core._local_browser_impl import LocalImpl -from dendrite.browser.async_api._remote_impl.browserbase._impl import BrowserBaseImpl -from dendrite.browser.async_api._remote_impl.browserless._impl import BrowserlessImpl from dendrite.browser.remote import Providers from dendrite.browser.remote.browserbase_config import BrowserbaseConfig from dendrite.browser.remote.browserless_config import BrowserlessConfig -IMPL_MAPPING: Dict[Type[Providers], Type[ImplBrowser]] = { - BrowserbaseConfig: BrowserBaseImpl, +from ..protocol.browser_protocol import BrowserProtocol +from .browserbase._impl import BrowserbaseImpl +from .browserless._impl import BrowserlessImpl +from .local._impl import LocalImpl + +IMPL_MAPPING: Dict[Type[Providers], Type[BrowserProtocol]] = { + BrowserbaseConfig: BrowserbaseImpl, BrowserlessConfig: BrowserlessImpl, - # BFloatProviderConfig: , } SETTINGS_CLASSES: Dict[str, Type[Providers]] = { @@ -20,7 +20,7 @@ } -def get_impl(remote_provider: Optional[Providers]) -> ImplBrowser: +def get_impl(remote_provider: Optional[Providers]) -> BrowserProtocol: if remote_provider is None: return LocalImpl() diff --git a/dendrite/browser/async_api/_core/_local_browser_impl.py b/dendrite/browser/async_api/browser_impl/local/_impl.py similarity index 80% rename from dendrite/browser/async_api/_core/_local_browser_impl.py rename to dendrite/browser/async_api/browser_impl/local/_impl.py index 009330c..ebc5010 100644 --- a/dendrite/browser/async_api/_core/_local_browser_impl.py +++ b/dendrite/browser/async_api/browser_impl/local/_impl.py @@ -1,30 +1,31 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional, Union, overload + from loguru import logger from typing_extensions import Literal from dendrite.browser._common.constants import STEALTH_ARGS if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from dendrite.browser.async_api.dendrite_browser import AsyncDendrite + +import os +import shutil +import tempfile from playwright.async_api import ( Browser, + BrowserContext, Download, Playwright, - BrowserContext, StorageState, ) -from dendrite.browser.async_api._core._impl_browser import ImplBrowser -from dendrite.browser.async_api._core._type_spec import PlaywrightPage - -import tempfile -import shutil -import os +from dendrite.browser.async_api.protocol.browser_protocol import BrowserProtocol +from dendrite.browser.async_api.types import PlaywrightPage -class LocalImpl(ImplBrowser): +class LocalImpl(BrowserProtocol): def __init__(self) -> None: pass diff --git a/dendrite/browser/async_api/_core/dendrite_browser.py b/dendrite/browser/async_api/dendrite_browser.py similarity index 96% rename from dendrite/browser/async_api/_core/dendrite_browser.py rename to dendrite/browser/async_api/dendrite_browser.py index bdf826a..8d6a4d3 100644 --- a/dendrite/browser/async_api/_core/dendrite_browser.py +++ b/dendrite/browser/async_api/dendrite_browser.py @@ -11,8 +11,8 @@ Error, FileChooser, FilePayload, - async_playwright, StorageState, + async_playwright, ) from dendrite.browser._common._exceptions.dendrite_exception import ( @@ -21,14 +21,16 @@ IncorrectOutcomeError, ) from dendrite.browser._common.constants import STEALTH_ARGS -from dendrite.browser.async_api._core._impl_browser import ImplBrowser -from dendrite.browser.async_api._core._impl_mapping import get_impl -from dendrite.browser.async_api._core._managers.page_manager import PageManager -from dendrite.browser.async_api._core._type_spec import PlaywrightPage -from dendrite.browser.async_api._core._utils import get_domain_w_suffix -from dendrite.browser.async_api._core.dendrite_page import AsyncPage -from dendrite.browser.async_api._core.event_sync import EventSync -from dendrite.browser.async_api._core.mixin import ( +from dendrite.browser.async_api._utils import get_domain_w_suffix +from dendrite.browser.remote import Providers +from dendrite.logic.config import Config +from dendrite.logic import AsyncLogicEngine + +from ._event_sync import EventSync +from .browser_impl.impl_mapping import get_impl +from .dendrite_page import AsyncPage +from .manager.page_manager import PageManager +from .mixin import ( AskMixin, ClickMixin, ExtractionMixin, @@ -39,9 +41,8 @@ ScreenshotMixin, WaitForMixin, ) -from dendrite.browser.remote import Providers -from dendrite.logic.config import Config -from dendrite.logic.interfaces import AsyncProtocol +from .protocol.browser_protocol import BrowserProtocol +from .types import PlaywrightPage class AsyncDendrite( @@ -110,7 +111,7 @@ def __init__( self._upload_handler = EventSync(event_type=FileChooser) self._download_handler = EventSync(event_type=Download) self.closed = False - self._browser_api_client: AsyncProtocol = AsyncProtocol(self._config) + self._browser_api_client: AsyncLogicEngine = AsyncLogicEngine(self._config) @property def pages(self) -> List[AsyncPage]: @@ -130,7 +131,7 @@ async def _get_page(self) -> AsyncPage: return active_page @property - def logic_engine(self) -> AsyncProtocol: + def logic_engine(self) -> AsyncLogicEngine: return self._browser_api_client @property @@ -144,7 +145,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): # Ensure cleanup is handled await self.close() - def _get_impl(self, remote_provider: Optional[Providers]) -> ImplBrowser: + def _get_impl(self, remote_provider: Optional[Providers]) -> BrowserProtocol: # if remote_provider is None:) return get_impl(remote_provider) diff --git a/dendrite/browser/async_api/_core/dendrite_element.py b/dendrite/browser/async_api/dendrite_element.py similarity index 96% rename from dendrite/browser/async_api/_core/dendrite_element.py rename to dendrite/browser/async_api/dendrite_element.py index cabec7f..bca3159 100644 --- a/dendrite/browser/async_api/_core/dendrite_element.py +++ b/dendrite/browser/async_api/dendrite_element.py @@ -12,19 +12,17 @@ from dendrite.browser._common._exceptions.dendrite_exception import ( IncorrectOutcomeError, ) - -from dendrite.logic.interfaces import AsyncProtocol +from dendrite.logic import AsyncLogicEngine if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from .dendrite_browser import AsyncDendrite -from dendrite.browser.async_api._core._managers.navigation_tracker import ( - NavigationTracker, -) -from dendrite.browser.async_api._core._type_spec import Interaction from dendrite.models.dto.make_interaction_dto import VerifyActionDTO from dendrite.models.response.interaction_response import InteractionResponse +from .manager.navigation_tracker import NavigationTracker +from .types import Interaction + def perform_action(interaction_type: Interaction): """ @@ -109,7 +107,7 @@ def __init__( dendrite_id: str, locator: Locator, dendrite_browser: AsyncDendrite, - browser_api_client: AsyncProtocol, + browser_api_client: AsyncLogicEngine, ): """ Initialize a AsyncElement. diff --git a/dendrite/browser/async_api/_core/dendrite_page.py b/dendrite/browser/async_api/dendrite_page.py similarity index 91% rename from dendrite/browser/async_api/_core/dendrite_page.py rename to dendrite/browser/async_api/dendrite_page.py index b770dad..f2b78e2 100644 --- a/dendrite/browser/async_api/_core/dendrite_page.py +++ b/dendrite/browser/async_api/dendrite_page.py @@ -8,29 +8,28 @@ from loguru import logger from playwright.async_api import Download, FilePayload, FrameLocator, Keyboard -from dendrite.browser.async_api._core._js import GENERATE_DENDRITE_IDS_SCRIPT -from dendrite.browser.async_api._core._type_spec import PlaywrightPage -from dendrite.browser.async_api._core.dendrite_element import AsyncElement -from dendrite.browser.async_api._core.mixin.ask import AskMixin -from dendrite.browser.async_api._core.mixin.click import ClickMixin -from dendrite.browser.async_api._core.mixin.extract import ExtractionMixin -from dendrite.browser.async_api._core.mixin.fill_fields import FillFieldsMixin -from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.async_api._core.mixin.keyboard import KeyboardMixin -from dendrite.browser.async_api._core.mixin.markdown import MarkdownMixin -from dendrite.browser.async_api._core.mixin.wait_for import WaitForMixin - -from dendrite.logic.interfaces import AsyncProtocol +from dendrite.logic import AsyncLogicEngine from dendrite.models.page_information import PageInformation +from .dendrite_element import AsyncElement +from .js import GENERATE_DENDRITE_IDS_SCRIPT +from .mixin.ask import AskMixin +from .mixin.click import ClickMixin +from .mixin.extract import ExtractionMixin +from .mixin.fill_fields import FillFieldsMixin +from .mixin.get_element import GetElementMixin +from .mixin.keyboard import KeyboardMixin +from .mixin.markdown import MarkdownMixin +from .mixin.wait_for import WaitForMixin +from .types import PlaywrightPage + if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from .dendrite_browser import AsyncDendrite from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.async_api._core._managers.screenshot_manager import ( - ScreenshotManager, -) -from dendrite.browser.async_api._core._utils import expand_iframes + +from ._utils import expand_iframes +from .manager.screenshot_manager import ScreenshotManager class AsyncPage( @@ -54,7 +53,7 @@ def __init__( self, page: PlaywrightPage, dendrite_browser: "AsyncDendrite", - browser_api_client: AsyncProtocol, + browser_api_client: AsyncLogicEngine, ): self.playwright_page = page self.screenshot_manager = ScreenshotManager(page) @@ -98,7 +97,7 @@ async def _get_page(self) -> "AsyncPage": return self @property - def logic_engine(self) -> AsyncProtocol: + def logic_engine(self) -> AsyncLogicEngine: return self._browser_api_client async def goto( diff --git a/dendrite/browser/async_api/_core/_js/__init__.py b/dendrite/browser/async_api/js/__init__.py similarity index 100% rename from dendrite/browser/async_api/_core/_js/__init__.py rename to dendrite/browser/async_api/js/__init__.py diff --git a/dendrite/browser/async_api/_core/_js/eventListenerPatch.js b/dendrite/browser/async_api/js/eventListenerPatch.js similarity index 100% rename from dendrite/browser/async_api/_core/_js/eventListenerPatch.js rename to dendrite/browser/async_api/js/eventListenerPatch.js diff --git a/dendrite/browser/async_api/_core/_js/generateDendriteIDs.js b/dendrite/browser/async_api/js/generateDendriteIDs.js similarity index 100% rename from dendrite/browser/async_api/_core/_js/generateDendriteIDs.js rename to dendrite/browser/async_api/js/generateDendriteIDs.js diff --git a/dendrite/browser/async_api/_core/_js/generateDendriteIDsIframe.js b/dendrite/browser/async_api/js/generateDendriteIDsIframe.js similarity index 100% rename from dendrite/browser/async_api/_core/_js/generateDendriteIDsIframe.js rename to dendrite/browser/async_api/js/generateDendriteIDsIframe.js diff --git a/dendrite/browser/async_api/_core/models/__init__.py b/dendrite/browser/async_api/manager/__init__.py similarity index 100% rename from dendrite/browser/async_api/_core/models/__init__.py rename to dendrite/browser/async_api/manager/__init__.py diff --git a/dendrite/browser/async_api/_core/_managers/navigation_tracker.py b/dendrite/browser/async_api/manager/navigation_tracker.py similarity index 97% rename from dendrite/browser/async_api/_core/_managers/navigation_tracker.py rename to dendrite/browser/async_api/manager/navigation_tracker.py index ac9b578..2ae51aa 100644 --- a/dendrite/browser/async_api/_core/_managers/navigation_tracker.py +++ b/dendrite/browser/async_api/manager/navigation_tracker.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_page import AsyncPage + from ..dendrite_page import AsyncPage class NavigationTracker: diff --git a/dendrite/browser/async_api/_core/_managers/page_manager.py b/dendrite/browser/async_api/manager/page_manager.py similarity index 94% rename from dendrite/browser/async_api/_core/_managers/page_manager.py rename to dendrite/browser/async_api/manager/page_manager.py index 25657f1..e9069af 100644 --- a/dendrite/browser/async_api/_core/_managers/page_manager.py +++ b/dendrite/browser/async_api/manager/page_manager.py @@ -4,10 +4,10 @@ from playwright.async_api import BrowserContext, Download, FileChooser if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from ..dendrite_browser import AsyncDendrite -from dendrite.browser.async_api._core._type_spec import PlaywrightPage -from dendrite.browser.async_api._core.dendrite_page import AsyncPage +from ..dendrite_page import AsyncPage +from ..types import PlaywrightPage class PageManager: diff --git a/dendrite/browser/async_api/_core/_managers/screenshot_manager.py b/dendrite/browser/async_api/manager/screenshot_manager.py similarity index 96% rename from dendrite/browser/async_api/_core/_managers/screenshot_manager.py rename to dendrite/browser/async_api/manager/screenshot_manager.py index a9cceb0..2c2613c 100644 --- a/dendrite/browser/async_api/_core/_managers/screenshot_manager.py +++ b/dendrite/browser/async_api/manager/screenshot_manager.py @@ -2,7 +2,7 @@ import os from uuid import uuid4 -from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from ..types import PlaywrightPage class ScreenshotManager: diff --git a/dendrite/browser/async_api/_core/mixin/__init__.py b/dendrite/browser/async_api/mixin/__init__.py similarity index 99% rename from dendrite/browser/async_api/_core/mixin/__init__.py rename to dendrite/browser/async_api/mixin/__init__.py index fd6b7cd..046a61c 100644 --- a/dendrite/browser/async_api/_core/mixin/__init__.py +++ b/dendrite/browser/async_api/mixin/__init__.py @@ -8,7 +8,6 @@ from .screenshot import ScreenshotMixin from .wait_for import WaitForMixin - __all__ = [ "AskMixin", "ClickMixin", diff --git a/dendrite/browser/async_api/_core/mixin/ask.py b/dendrite/browser/async_api/mixin/ask.py similarity index 96% rename from dendrite/browser/async_api/_core/mixin/ask.py rename to dendrite/browser/async_api/mixin/ask.py index 8f86e5c..b7efd7a 100644 --- a/dendrite/browser/async_api/_core/mixin/ask.py +++ b/dendrite/browser/async_api/mixin/ask.py @@ -5,16 +5,12 @@ from loguru import logger from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.async_api._core._type_spec import ( - JsonSchema, - PydanticModel, - TypeSpec, - convert_to_type_spec, - to_json_schema, -) -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.async_api._utils import convert_to_type_spec, to_json_schema from dendrite.models.dto.ask_page_dto import AskPageDTO +from ..protocol.page_protocol import DendritePageProtocol +from ..types import JsonSchema, PydanticModel, TypeSpec + # The timeout interval between retries in milliseconds TIMEOUT_INTERVAL = [150, 450, 1000] diff --git a/dendrite/browser/async_api/_core/mixin/click.py b/dendrite/browser/async_api/mixin/click.py similarity index 92% rename from dendrite/browser/async_api/_core/mixin/click.py rename to dendrite/browser/async_api/mixin/click.py index 1b3b110..d6460f2 100644 --- a/dendrite/browser/async_api/_core/mixin/click.py +++ b/dendrite/browser/async_api/mixin/click.py @@ -1,10 +1,11 @@ from typing import Optional from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol from dendrite.models.response.interaction_response import InteractionResponse +from ..mixin.get_element import GetElementMixin +from ..protocol.page_protocol import DendritePageProtocol + class ClickMixin(GetElementMixin, DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/mixin/extract.py b/dendrite/browser/async_api/mixin/extract.py similarity index 96% rename from dendrite/browser/async_api/_core/mixin/extract.py rename to dendrite/browser/async_api/mixin/extract.py index 093de34..f3ec334 100644 --- a/dendrite/browser/async_api/_core/mixin/extract.py +++ b/dendrite/browser/async_api/mixin/extract.py @@ -4,23 +4,17 @@ from loguru import logger -from dendrite.browser.async_api._core._managers.navigation_tracker import ( - NavigationTracker, -) -from dendrite.browser.async_api._core._type_spec import ( - JsonSchema, - PydanticModel, - TypeSpec, - convert_to_type_spec, - to_json_schema, -) -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from dendrite.browser.async_api._utils import convert_to_type_spec, to_json_schema from dendrite.logic.code.code_session import execute from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.dto.extract_dto import ExtractDTO from dendrite.models.response.extract_response import ExtractResponse from dendrite.models.scripts import Script +from ..manager.navigation_tracker import NavigationTracker +from ..protocol.page_protocol import DendritePageProtocol +from ..types import JsonSchema, PydanticModel, TypeSpec + CACHE_TIMEOUT = 5 diff --git a/dendrite/browser/async_api/_core/mixin/fill_fields.py b/dendrite/browser/async_api/mixin/fill_fields.py similarity index 95% rename from dendrite/browser/async_api/_core/mixin/fill_fields.py rename to dendrite/browser/async_api/mixin/fill_fields.py index cb2701a..fad759f 100644 --- a/dendrite/browser/async_api/_core/mixin/fill_fields.py +++ b/dendrite/browser/async_api/mixin/fill_fields.py @@ -2,10 +2,11 @@ from typing import Any, Dict, Optional from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.async_api._core.mixin.get_element import GetElementMixin -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol from dendrite.models.response.interaction_response import InteractionResponse +from ..mixin.get_element import GetElementMixin +from ..protocol.page_protocol import DendritePageProtocol + class FillFieldsMixin(GetElementMixin, DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/mixin/get_element.py b/dendrite/browser/async_api/mixin/get_element.py similarity index 68% rename from dendrite/browser/async_api/_core/mixin/get_element.py rename to dendrite/browser/async_api/mixin/get_element.py index a488f08..44878a9 100644 --- a/dendrite/browser/async_api/_core/mixin/get_element.py +++ b/dendrite/browser/async_api/mixin/get_element.py @@ -15,102 +15,21 @@ from bs4 import BeautifulSoup from loguru import logger -from dendrite.browser.async_api._core._utils import _get_all_elements_from_selector_soup -from dendrite.browser.async_api._core.dendrite_element import AsyncElement +from .._utils import _get_all_elements_from_selector_soup +from ..dendrite_element import AsyncElement if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_page import AsyncPage -from dendrite.browser.async_api._core.models.response import AsyncElementsResponse -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol + from ..dendrite_page import AsyncPage + from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO from dendrite.models.dto.get_elements_dto import GetElementsDTO +from ..protocol.page_protocol import DendritePageProtocol CACHE_TIMEOUT = 5 class GetElementMixin(DendritePageProtocol): - @overload - async def get_elements( - self, - prompt_or_elements: str, - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> List[AsyncElement]: - """ - Retrieves a list of Dendrite elements based on a string prompt. - - Args: - prompt_or_elements (str): The prompt describing the elements to be retrieved. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - List[AsyncElement]: A list of Dendrite elements found on the page. - """ - - @overload - async def get_elements( - self, - prompt_or_elements: Dict[str, str], - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> AsyncElementsResponse: - """ - Retrieves Dendrite elements based on a dictionary. - - Args: - prompt_or_elements (Dict[str, str]): A dictionary where keys are field names and values are prompts describing the elements to be retrieved. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - AsyncElementsResponse: A response object containing the retrieved elements with attributes matching the keys in the dict. - """ - - async def get_elements( - self, - prompt_or_elements: Union[str, Dict[str, str]], - use_cache: bool = True, - timeout: int = 15000, - context: str = "", - ) -> Union[List[AsyncElement], AsyncElementsResponse]: - """ - Retrieves Dendrite elements based on either a string prompt or a dictionary of prompts. - - This method determines the type of the input (string or dictionary) and retrieves the appropriate elements. - If the input is a string, it fetches a list of elements. If the input is a dictionary, it fetches elements for each key-value pair. - - Args: - prompt_or_elements (Union[str, Dict[str, str]]): The prompt or dictionary of prompts for element retrieval. - use_cache (bool, optional): Whether to use cached results. Defaults to True. - timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, - up to 5000ms will be spent attempting to use cached selectors before falling back to the - find element agent for the remaining time. Defaults to 15000 (15 seconds). - context (str, optional): Additional context for the retrieval. Defaults to an empty string. - - Returns: - Union[List[AsyncElement], AsyncElementsResponse]: A list of elements or a response object containing the retrieved elements. - - Raises: - ValueError: If the input is neither a string nor a dictionary. - """ - - return await self._get_element( - prompt_or_elements, - only_one=False, - use_cache=use_cache, - timeout=timeout / 1000, - ) - async def get_element( self, prompt: str, @@ -163,11 +82,11 @@ async def _get_element( @overload async def _get_element( self, - prompt_or_elements: Union[str, Dict[str, str]], + prompt_or_elements: str, only_one: Literal[False], use_cache: bool, timeout, - ) -> Union[List[AsyncElement], AsyncElementsResponse]: + ) -> List[AsyncElement]: """ Retrieves a list of Dendrite elements based on the provided prompt. @@ -185,14 +104,13 @@ async def _get_element( async def _get_element( self, - prompt_or_elements: Union[str, Dict[str, str]], + prompt_or_elements: str, only_one: bool, use_cache: bool, timeout: float, ) -> Union[ Optional[AsyncElement], List[AsyncElement], - AsyncElementsResponse, ]: """ Retrieves Dendrite elements based on the provided prompt, either a single element or a list of elements. @@ -211,9 +129,6 @@ async def _get_element( Union[AsyncElement, List[AsyncElement], AsyncElementsResponse]: The retrieved element, list of elements, or response object. """ - if isinstance(prompt_or_elements, Dict): - return None - logger.info(f"Getting element for prompt: '{prompt_or_elements}'") start_time = time.time() page = await self._get_page() @@ -336,7 +251,7 @@ async def try_get_element( prompt_or_elements: Union[str, Dict[str, str]], only_one: bool, remaining_timeout: float, -) -> Union[Optional[AsyncElement], List[AsyncElement], AsyncElementsResponse]: +) -> Union[Optional[AsyncElement], List[AsyncElement]]: async def _try_get_element(): page = await obj._get_page() diff --git a/dendrite/browser/async_api/_core/mixin/keyboard.py b/dendrite/browser/async_api/mixin/keyboard.py similarity index 95% rename from dendrite/browser/async_api/_core/mixin/keyboard.py rename to dendrite/browser/async_api/mixin/keyboard.py index 008c464..bf4e145 100644 --- a/dendrite/browser/async_api/_core/mixin/keyboard.py +++ b/dendrite/browser/async_api/mixin/keyboard.py @@ -1,7 +1,8 @@ from typing import Literal, Union from dendrite.browser._common._exceptions.dendrite_exception import DendriteException -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol + +from ..protocol.page_protocol import DendritePageProtocol class KeyboardMixin(DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/mixin/markdown.py b/dendrite/browser/async_api/mixin/markdown.py similarity index 89% rename from dendrite/browser/async_api/_core/mixin/markdown.py rename to dendrite/browser/async_api/mixin/markdown.py index 8bd1697..687db67 100644 --- a/dendrite/browser/async_api/_core/mixin/markdown.py +++ b/dendrite/browser/async_api/mixin/markdown.py @@ -4,8 +4,8 @@ from bs4 import BeautifulSoup from markdownify import markdownify as md -from dendrite.browser.async_api._core.mixin.extract import ExtractionMixin -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from ..mixin.extract import ExtractionMixin +from ..protocol.page_protocol import DendritePageProtocol class MarkdownMixin(ExtractionMixin, DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/mixin/screenshot.py b/dendrite/browser/async_api/mixin/screenshot.py similarity index 87% rename from dendrite/browser/async_api/_core/mixin/screenshot.py rename to dendrite/browser/async_api/mixin/screenshot.py index 9877319..200d4a1 100644 --- a/dendrite/browser/async_api/_core/mixin/screenshot.py +++ b/dendrite/browser/async_api/mixin/screenshot.py @@ -1,4 +1,4 @@ -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol +from ..protocol.page_protocol import DendritePageProtocol class ScreenshotMixin(DendritePageProtocol): diff --git a/dendrite/browser/async_api/_core/mixin/wait_for.py b/dendrite/browser/async_api/mixin/wait_for.py similarity index 94% rename from dendrite/browser/async_api/_core/mixin/wait_for.py rename to dendrite/browser/async_api/mixin/wait_for.py index 58ffacf..7c60f88 100644 --- a/dendrite/browser/async_api/_core/mixin/wait_for.py +++ b/dendrite/browser/async_api/mixin/wait_for.py @@ -7,8 +7,9 @@ DendriteException, PageConditionNotMet, ) -from dendrite.browser.async_api._core.mixin.ask import AskMixin -from dendrite.browser.async_api._core.protocol.page_protocol import DendritePageProtocol + +from ..mixin.ask import AskMixin +from ..protocol.page_protocol import DendritePageProtocol class WaitForMixin(AskMixin, DendritePageProtocol): diff --git a/dendrite/browser/async_api/_dom/__init__.py b/dendrite/browser/async_api/protocol/__init__.py similarity index 100% rename from dendrite/browser/async_api/_dom/__init__.py rename to dendrite/browser/async_api/protocol/__init__.py diff --git a/dendrite/browser/async_api/_core/_impl_browser.py b/dendrite/browser/async_api/protocol/browser_protocol.py similarity index 69% rename from dendrite/browser/async_api/_core/_impl_browser.py rename to dendrite/browser/async_api/protocol/browser_protocol.py index 9d9aff3..304064d 100644 --- a/dendrite/browser/async_api/_core/_impl_browser.py +++ b/dendrite/browser/async_api/protocol/browser_protocol.py @@ -1,28 +1,20 @@ -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Optional, Union, overload +from typing import TYPE_CHECKING, Optional, Protocol, Union + from typing_extensions import Literal +from dendrite.browser.remote import Providers + if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite + from ..dendrite_browser import AsyncDendrite -from playwright.async_api import ( - Browser, - Download, - Playwright, - BrowserContext, - StorageState, -) +from playwright.async_api import Browser, Download, Playwright -from dendrite.browser.async_api._core._type_spec import PlaywrightPage +from ..types import PlaywrightPage -class ImplBrowser(ABC): - @abstractmethod - def __init__(self, settings): - pass - # self.settings = settings +class BrowserProtocol(Protocol): + def __init__(self, settings: Providers) -> None: ... - @abstractmethod async def get_download( self, dendrite_browser: "AsyncDendrite", pw_page: PlaywrightPage, timeout: float ) -> Download: @@ -35,8 +27,8 @@ async def get_download( Raises: Exception: If there is an issue retrieving the download event. """ + ... - @abstractmethod async def start_browser( self, playwright: Playwright, @@ -52,8 +44,8 @@ async def start_browser( Returns: Browser: A Browser instance """ + ... - @abstractmethod async def configure_context(self, browser: "AsyncDendrite") -> None: """ Configures the browser context. @@ -64,8 +56,8 @@ async def configure_context(self, browser: "AsyncDendrite") -> None: Raises: Exception: If there is an issue configuring the browser context. """ + ... - @abstractmethod async def stop_session(self) -> None: """ Stops the browser session. @@ -73,3 +65,4 @@ async def stop_session(self) -> None: Raises: Exception: If there is an issue stopping the browser session. """ + ... diff --git a/dendrite/browser/async_api/_core/models/download_interface.py b/dendrite/browser/async_api/protocol/download_protocol.py similarity index 100% rename from dendrite/browser/async_api/_core/models/download_interface.py rename to dendrite/browser/async_api/protocol/download_protocol.py diff --git a/dendrite/browser/async_api/_core/protocol/page_protocol.py b/dendrite/browser/async_api/protocol/page_protocol.py similarity index 58% rename from dendrite/browser/async_api/_core/protocol/page_protocol.py rename to dendrite/browser/async_api/protocol/page_protocol.py index 920434d..7716352 100644 --- a/dendrite/browser/async_api/_core/protocol/page_protocol.py +++ b/dendrite/browser/async_api/protocol/page_protocol.py @@ -1,10 +1,10 @@ from typing import TYPE_CHECKING, Protocol -from dendrite.logic.interfaces import AsyncProtocol +from dendrite.logic import AsyncLogicEngine if TYPE_CHECKING: - from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite - from dendrite.browser.async_api._core.dendrite_page import AsyncPage + from ..dendrite_browser import AsyncDendrite + from ..dendrite_page import AsyncPage class DendritePageProtocol(Protocol): @@ -14,7 +14,7 @@ class DendritePageProtocol(Protocol): """ @property - def logic_engine(self) -> AsyncProtocol: ... + def logic_engine(self) -> AsyncLogicEngine: ... @property def dendrite_browser(self) -> "AsyncDendrite": ... diff --git a/dendrite/browser/async_api/types.py b/dendrite/browser/async_api/types.py new file mode 100644 index 0000000..1703b8c --- /dev/null +++ b/dendrite/browser/async_api/types.py @@ -0,0 +1,15 @@ +import inspect +from typing import Any, Dict, Literal, Type, TypeVar, Union + +from playwright.async_api import Page +from pydantic import BaseModel + +Interaction = Literal["click", "fill", "hover"] + +T = TypeVar("T") +PydanticModel = TypeVar("PydanticModel", bound=BaseModel) +PrimitiveTypes = PrimitiveTypes = Union[Type[bool], Type[int], Type[float], Type[str]] +JsonSchema = Dict[str, Any] +TypeSpec = Union[PrimitiveTypes, PydanticModel, JsonSchema] + +PlaywrightPage = Page diff --git a/dendrite/browser/sync_api/__init__.py b/dendrite/browser/sync_api/__init__.py new file mode 100644 index 0000000..8beebcc --- /dev/null +++ b/dendrite/browser/sync_api/__init__.py @@ -0,0 +1,6 @@ +from loguru import logger +from .dendrite_browser import Dendrite +from .dendrite_element import Element +from .dendrite_page import Page + +__all__ = ["Dendrite", "Element", "Page"] diff --git a/dendrite/browser/sync_api/_event_sync.py b/dendrite/browser/sync_api/_event_sync.py new file mode 100644 index 0000000..4351eee --- /dev/null +++ b/dendrite/browser/sync_api/_event_sync.py @@ -0,0 +1,45 @@ +import time +import time +from typing import Generic, Optional, Type, TypeVar +from playwright.sync_api import Download, FileChooser, Page + +Events = TypeVar("Events", Download, FileChooser) +mapping = {Download: "download", FileChooser: "filechooser"} + + +class EventSync(Generic[Events]): + + def __init__(self, event_type: Type[Events]): + self.event_type = event_type + self.event_set = False + self.data: Optional[Events] = None + + def get_data(self, pw_page: Page, timeout: float = 30000) -> Events: + start_time = time.time() + while not self.event_set: + elapsed_time = (time.time() - start_time) * 1000 + if elapsed_time > timeout: + raise TimeoutError(f'Timeout waiting for event "{self.event_type}".') + pw_page.wait_for_timeout(0) + time.sleep(0.01) + data = self.data + self.data = None + self.event_set = False + if data is None: + raise ValueError("Data is None for event type: ", self.event_type) + return data + + def set_event(self, data: Events) -> None: + """ + Sets the event and stores the provided data. + + This method is used to signal that the data is ready to be retrieved by any waiting tasks. + + Args: + data (T): The data to be stored and associated with the event. + + Returns: + None + """ + self.data = data + self.event_set = True diff --git a/dendrite/browser/sync_api/_utils.py b/dendrite/browser/sync_api/_utils.py new file mode 100644 index 0000000..eecf191 --- /dev/null +++ b/dendrite/browser/sync_api/_utils.py @@ -0,0 +1,124 @@ +import inspect +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +import tldextract +from bs4 import BeautifulSoup +from loguru import logger +from playwright.sync_api import Error, Frame +from pydantic import BaseModel +from dendrite.models.selector import Selector +from .dendrite_element import Element +from .types import PlaywrightPage, TypeSpec + +if TYPE_CHECKING: + from .dendrite_page import Page +from dendrite.logic.dom.strip import mild_strip_in_place +from .js import GENERATE_DENDRITE_IDS_IFRAME_SCRIPT + + +def get_domain_w_suffix(url: str) -> str: + parsed_url = tldextract.extract(url) + if parsed_url.suffix == "": + raise ValueError(f"Invalid URL: {url}") + return f"{parsed_url.domain}.{parsed_url.suffix}" + + +def expand_iframes(page: PlaywrightPage, page_soup: BeautifulSoup): + + def get_iframe_path(frame: Frame): + path_parts = [] + current_frame = frame + while current_frame.parent_frame is not None: + iframe_element = current_frame.frame_element() + iframe_id = iframe_element.get_attribute("d-id") + if iframe_id is None: + return None + path_parts.insert(0, iframe_id) + current_frame = current_frame.parent_frame + return "|".join(path_parts) + + for frame in page.frames: + if frame.parent_frame is None: + continue + try: + iframe_element = frame.frame_element() + iframe_id = iframe_element.get_attribute("d-id") + if iframe_id is None: + continue + iframe_path = get_iframe_path(frame) + except Error as e: + continue + if iframe_path is None: + continue + try: + frame.evaluate( + GENERATE_DENDRITE_IDS_IFRAME_SCRIPT, {"frame_path": iframe_path} + ) + frame_content = frame.content() + frame_tree = BeautifulSoup(frame_content, "lxml") + mild_strip_in_place(frame_tree) + merge_iframe_to_page(iframe_id, page_soup, frame_tree) + except Error as e: + logger.debug(f"Error processing frame {iframe_id}: {e}") + continue + + +def merge_iframe_to_page(iframe_id: str, page: BeautifulSoup, iframe: BeautifulSoup): + iframe_element = page.find("iframe", {"d-id": iframe_id}) + if iframe_element is None: + logger.debug(f"Could not find iframe with ID {iframe_id} in page soup") + return + iframe_element.replace_with(iframe) + + +def _get_all_elements_from_selector_soup( + selector: str, soup: BeautifulSoup, page: "Page" +) -> List[Element]: + dendrite_elements: List[Element] = [] + elements = soup.select(selector) + for element in elements: + frame = page._get_context(element) + d_id = element.get("d-id", "") + locator = frame.locator(f"xpath=//*[@d-id='{d_id}']") + if not d_id: + continue + if isinstance(d_id, list): + d_id = d_id[0] + dendrite_elements.append( + Element(d_id, locator, page.dendrite_browser, page._browser_api_client) + ) + return dendrite_elements + + +def get_elements_from_selectors_soup( + page: "Page", soup: BeautifulSoup, selectors: List[Selector], only_one: bool +) -> Union[Optional[Element], List[Element]]: + for selector in reversed(selectors): + dendrite_elements = _get_all_elements_from_selector_soup( + selector.selector, soup, page + ) + if len(dendrite_elements) > 0: + return dendrite_elements[0] if only_one else dendrite_elements + return None + + +def to_json_schema(type_spec: TypeSpec) -> Dict[str, Any]: + if isinstance(type_spec, dict): + return type_spec + if inspect.isclass(type_spec) and issubclass(type_spec, BaseModel): + return type_spec.model_json_schema() + if type_spec in (bool, int, float, str): + type_map = {bool: "boolean", int: "integer", float: "number", str: "string"} + return {"type": type_map[type_spec]} + raise ValueError(f"Unsupported type specification: {type_spec}") + + +def convert_to_type_spec(type_spec: TypeSpec, return_data: Any) -> TypeSpec: + if isinstance(type_spec, type): + if issubclass(type_spec, BaseModel): + return type_spec.model_validate(return_data) + if type_spec in (str, float, bool, int): + return type_spec(return_data) + raise ValueError(f"Unsupported type: {type_spec}") + if isinstance(type_spec, dict): + return return_data + raise ValueError(f"Unsupported type specification: {type_spec}") diff --git a/dendrite/browser/sync_api/browser_impl/__init__.py b/dendrite/browser/sync_api/browser_impl/__init__.py new file mode 100644 index 0000000..4d00d3c --- /dev/null +++ b/dendrite/browser/sync_api/browser_impl/__init__.py @@ -0,0 +1,3 @@ +from .browserbase import BrowserbaseDownload + +__all__ = ["BrowserbaseDownload"] diff --git a/dendrite/browser/sync_api/browser_impl/browserbase/__init__.py b/dendrite/browser/sync_api/browser_impl/browserbase/__init__.py new file mode 100644 index 0000000..eb977c7 --- /dev/null +++ b/dendrite/browser/sync_api/browser_impl/browserbase/__init__.py @@ -0,0 +1,3 @@ +from ._download import BrowserbaseDownload + +__all__ = ["BrowserbaseDownload"] diff --git a/dendrite/browser/sync_api/browser_impl/browserbase/_client.py b/dendrite/browser/sync_api/browser_impl/browserbase/_client.py new file mode 100644 index 0000000..ddc6831 --- /dev/null +++ b/dendrite/browser/sync_api/browser_impl/browserbase/_client.py @@ -0,0 +1,63 @@ +import time +import time +from pathlib import Path +from typing import Optional, Union +import httpx +from loguru import logger +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException + + +class BrowserbaseClient: + + def __init__(self, api_key: str, project_id: str) -> None: + self.api_key = api_key + self.project_id = project_id + + def create_session(self) -> str: + logger.debug("Creating session") + "\n Creates a session using the Browserbase API.\n\n Returns:\n str: The ID of the created session.\n " + url = "https://www.browserbase.com/v1/sessions" + headers = {"Content-Type": "application/json", "x-bb-api-key": self.api_key} + json = {"projectId": self.project_id, "keepAlive": False} + response = httpx.post(url, json=json, headers=headers) + if response.status_code >= 400: + raise DendriteException(f"Failed to create session: {response.text}") + return response.json()["id"] + + def stop_session(self, session_id: str): + url = f"https://www.browserbase.com/v1/sessions/{session_id}" + headers = {"Content-Type": "application/json", "x-bb-api-key": self.api_key} + json = {"projectId": self.project_id, "status": "REQUEST_RELEASE"} + with httpx.Client() as client: + response = client.post(url, json=json, headers=headers) + return response.json() + + def connect_url(self, enable_proxy: bool, session_id: Optional[str] = None) -> str: + url = f"wss://connect.browserbase.com?apiKey={self.api_key}" + if session_id: + url += f"&sessionId={session_id}" + if enable_proxy: + url += "&enableProxy=true" + return url + + def save_downloads_on_disk( + self, session_id: str, path: Union[str, Path], retry_for_seconds: float + ): + url = f"https://www.browserbase.com/v1/sessions/{session_id}/downloads" + headers = {"x-bb-api-key": self.api_key} + file_path = Path(path) + with httpx.Client() as session: + timeout = time.time() + retry_for_seconds + while time.time() < timeout: + try: + response = session.get(url, headers=headers) + if response.status_code == 200: + array_buffer = response.read() + if len(array_buffer) > 0: + with open(file_path, "wb") as f: + f.write(array_buffer) + return + except Exception as e: + logger.debug(f"Error fetching downloads: {e}") + time.sleep(2) + logger.debug("Failed to download files within the time limit.") diff --git a/dendrite/browser/sync_api/browser_impl/browserbase/_download.py b/dendrite/browser/sync_api/browser_impl/browserbase/_download.py new file mode 100644 index 0000000..c464c81 --- /dev/null +++ b/dendrite/browser/sync_api/browser_impl/browserbase/_download.py @@ -0,0 +1,53 @@ +import re +import shutil +import zipfile +from pathlib import Path +from typing import Union +from loguru import logger +from playwright.sync_api import Download +from dendrite.browser.sync_api.browser_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.sync_api.protocol.download_protocol import DownloadInterface + + +class BrowserbaseDownload(DownloadInterface): + + def __init__( + self, session_id: str, download: Download, client: BrowserbaseClient + ) -> None: + super().__init__(download) + self._session_id = session_id + self._client = client + + def save_as(self, path: Union[str, Path], timeout: float = 20) -> None: + """ + Save the latest file from the downloaded ZIP archive to the specified path. + + Args: + path (Union[str, Path]): The destination file path where the latest file will be saved. + timeout (float, optional): Timeout for the save operation. Defaults to 20 seconds. + + Raises: + Exception: If no matching files are found in the ZIP archive or if the file cannot be saved. + """ + destination_path = Path(path) + source_path = self._download.path() + destination_path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(source_path, "r") as zip_ref: + file_list = zip_ref.namelist() + sorted_files = sorted(file_list, key=extract_timestamp, reverse=True) + if not sorted_files: + raise FileNotFoundError( + "No files found in the Browserbase download ZIP" + ) + latest_file = sorted_files[0] + with zip_ref.open(latest_file) as source, open( + destination_path, "wb" + ) as target: + shutil.copyfileobj(source, target) + logger.info(f"Latest file saved successfully to {destination_path}") + + +def extract_timestamp(filename): + timestamp_pattern = re.compile("-(\\d+)\\.") + match = timestamp_pattern.search(filename) + return int(match.group(1)) if match else 0 diff --git a/dendrite/browser/sync_api/browser_impl/browserbase/_impl.py b/dendrite/browser/sync_api/browser_impl/browserbase/_impl.py new file mode 100644 index 0000000..60ceaf3 --- /dev/null +++ b/dendrite/browser/sync_api/browser_impl/browserbase/_impl.py @@ -0,0 +1,62 @@ +from typing import TYPE_CHECKING, Optional +from dendrite.browser._common._exceptions.dendrite_exception import ( + BrowserNotLaunchedError, +) +from dendrite.browser.sync_api.protocol.browser_protocol import BrowserProtocol +from dendrite.browser.sync_api.types import PlaywrightPage +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig + +if TYPE_CHECKING: + from dendrite.browser.sync_api.dendrite_browser import Dendrite +from loguru import logger +from playwright.sync_api import Playwright +from ._client import BrowserbaseClient +from ._download import BrowserbaseDownload + + +class BrowserbaseImpl(BrowserProtocol): + + def __init__(self, settings: BrowserbaseConfig) -> None: + self.settings = settings + self._client = BrowserbaseClient( + self.settings.api_key, self.settings.project_id + ) + self._session_id: Optional[str] = None + + def stop_session(self): + if self._session_id: + self._client.stop_session(self._session_id) + + def start_browser(self, playwright: Playwright, pw_options: dict): + logger.debug("Starting browser") + self._session_id = self._client.create_session() + url = self._client.connect_url(self.settings.enable_proxy, self._session_id) + logger.debug(f"Connecting to browser at {url}") + return playwright.chromium.connect_over_cdp(url) + + def configure_context(self, browser: "Dendrite"): + logger.debug("Configuring browser context") + page = browser.get_active_page() + pw_page = page.playwright_page + if browser.browser_context is None: + raise BrowserNotLaunchedError() + client = browser.browser_context.new_cdp_session(pw_page) + client.send( + "Browser.setDownloadBehavior", + {"behavior": "allow", "downloadPath": "downloads", "eventsEnabled": True}, + ) + + def get_download( + self, + dendrite_browser: "Dendrite", + pw_page: PlaywrightPage, + timeout: float = 30000, + ) -> BrowserbaseDownload: + if not self._session_id: + raise ValueError( + "Downloads are not enabled for this provider. Specify enable_downloads=True in the constructor" + ) + logger.debug("Getting download") + download = dendrite_browser._download_handler.get_data(pw_page, timeout) + self._client.save_downloads_on_disk(self._session_id, download.path(), 30) + return BrowserbaseDownload(self._session_id, download, self._client) diff --git a/dendrite/browser/sync_api/browser_impl/browserless/__init__.py b/dendrite/browser/sync_api/browser_impl/browserless/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/browser_impl/browserless/_impl.py b/dendrite/browser/sync_api/browser_impl/browserless/_impl.py new file mode 100644 index 0000000..822ed48 --- /dev/null +++ b/dendrite/browser/sync_api/browser_impl/browserless/_impl.py @@ -0,0 +1,57 @@ +import json +from typing import TYPE_CHECKING, Optional +from dendrite.browser._common._exceptions.dendrite_exception import ( + BrowserNotLaunchedError, +) +from dendrite.browser.sync_api.protocol.browser_protocol import BrowserProtocol +from dendrite.browser.sync_api.types import PlaywrightPage +from dendrite.browser.remote.browserless_config import BrowserlessConfig + +if TYPE_CHECKING: + from dendrite.browser.sync_api.dendrite_browser import Dendrite +import urllib.parse +from loguru import logger +from playwright.sync_api import Playwright +from dendrite.browser.sync_api.browser_impl.browserbase._client import BrowserbaseClient +from dendrite.browser.sync_api.browser_impl.browserbase._download import ( + BrowserbaseDownload, +) + + +class BrowserlessImpl(BrowserProtocol): + + def __init__(self, settings: BrowserlessConfig) -> None: + self.settings = settings + self._session_id: Optional[str] = None + + def stop_session(self): + pass + + def start_browser(self, playwright: Playwright, pw_options: dict): + logger.debug("Starting browser") + url = self._format_connection_url(pw_options) + logger.debug(f"Connecting to browser at {url}") + return playwright.chromium.connect_over_cdp(url) + + def _format_connection_url(self, pw_options: dict) -> str: + url = self.settings.url.rstrip("?").rstrip("/") + query = { + "token": self.settings.api_key, + "blockAds": self.settings.block_ads, + "launch": json.dumps(pw_options), + } + if self.settings.proxy: + query["proxy"] = (self.settings.proxy,) + query["proxyCountry"] = (self.settings.proxy_country,) + return f"{url}?{urllib.parse.urlencode(query)}" + + def configure_context(self, browser: "Dendrite"): + pass + + def get_download( + self, + dendrite_browser: "Dendrite", + pw_page: PlaywrightPage, + timeout: float = 30000, + ) -> BrowserbaseDownload: + raise NotImplementedError("Downloads are not supported for Browserless") diff --git a/dendrite/browser/sync_api/browser_impl/impl_mapping.py b/dendrite/browser/sync_api/browser_impl/impl_mapping.py new file mode 100644 index 0000000..d1e3d65 --- /dev/null +++ b/dendrite/browser/sync_api/browser_impl/impl_mapping.py @@ -0,0 +1,29 @@ +from typing import Dict, Optional, Type +from dendrite.browser.remote import Providers +from dendrite.browser.remote.browserbase_config import BrowserbaseConfig +from dendrite.browser.remote.browserless_config import BrowserlessConfig +from ..protocol.browser_protocol import BrowserProtocol +from .browserbase._impl import BrowserbaseImpl +from .browserless._impl import BrowserlessImpl +from .local._impl import LocalImpl + +IMPL_MAPPING: Dict[Type[Providers], Type[BrowserProtocol]] = { + BrowserbaseConfig: BrowserbaseImpl, + BrowserlessConfig: BrowserlessImpl, +} +SETTINGS_CLASSES: Dict[str, Type[Providers]] = { + "browserbase": BrowserbaseConfig, + "browserless": BrowserlessConfig, +} + + +def get_impl(remote_provider: Optional[Providers]) -> BrowserProtocol: + if remote_provider is None: + return LocalImpl() + try: + provider_class = IMPL_MAPPING[type(remote_provider)] + except KeyError: + raise ValueError( + f"No implementation for {type(remote_provider)}. Available providers: {', '.join(map(lambda x: x.__name__, IMPL_MAPPING.keys()))}" + ) + return provider_class(remote_provider) diff --git a/dendrite/browser/sync_api/browser_impl/local/__init__.py b/dendrite/browser/sync_api/browser_impl/local/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/browser_impl/local/_impl.py b/dendrite/browser/sync_api/browser_impl/local/_impl.py new file mode 100644 index 0000000..e995cc1 --- /dev/null +++ b/dendrite/browser/sync_api/browser_impl/local/_impl.py @@ -0,0 +1,45 @@ +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union, overload +from loguru import logger +from typing_extensions import Literal +from dendrite.browser._common.constants import STEALTH_ARGS + +if TYPE_CHECKING: + from dendrite.browser.sync_api.dendrite_browser import Dendrite +import os +import shutil +import tempfile +from playwright.sync_api import ( + Browser, + BrowserContext, + Download, + Playwright, + StorageState, +) +from dendrite.browser.sync_api.protocol.browser_protocol import BrowserProtocol +from dendrite.browser.sync_api.types import PlaywrightPage + + +class LocalImpl(BrowserProtocol): + + def __init__(self) -> None: + pass + + def start_browser( + self, + playwright: Playwright, + pw_options: dict, + storage_state: Optional[StorageState] = None, + ) -> Browser: + return playwright.chromium.launch(**pw_options) + + def get_download( + self, dendrite_browser: "Dendrite", pw_page: PlaywrightPage, timeout: float + ) -> Download: + return dendrite_browser._download_handler.get_data(pw_page, timeout) + + def configure_context(self, browser: "Dendrite"): + pass + + def stop_session(self): + pass diff --git a/dendrite/browser/sync_api/dendrite_browser.py b/dendrite/browser/sync_api/dendrite_browser.py new file mode 100644 index 0000000..b5fdd64 --- /dev/null +++ b/dendrite/browser/sync_api/dendrite_browser.py @@ -0,0 +1,485 @@ +import os +import pathlib +import re +from abc import ABC +from typing import Any, List, Optional, Sequence, Union +from uuid import uuid4 +from loguru import logger +from playwright.sync_api import ( + Download, + Error, + FileChooser, + FilePayload, + StorageState, + sync_playwright, +) +from dendrite.browser._common._exceptions.dendrite_exception import ( + BrowserNotLaunchedError, + DendriteException, + IncorrectOutcomeError, +) +from dendrite.browser._common.constants import STEALTH_ARGS +from dendrite.browser.sync_api._utils import get_domain_w_suffix +from dendrite.browser.remote import Providers +from dendrite.logic.config import Config +from dendrite.logic import LogicEngine +from ._event_sync import EventSync +from .browser_impl.impl_mapping import get_impl +from .dendrite_page import Page +from .manager.page_manager import PageManager +from .mixin import ( + AskMixin, + ClickMixin, + ExtractionMixin, + FillFieldsMixin, + GetElementMixin, + KeyboardMixin, + MarkdownMixin, + ScreenshotMixin, + WaitForMixin, +) +from .protocol.browser_protocol import BrowserProtocol +from .types import PlaywrightPage + + +class Dendrite( + ScreenshotMixin, + WaitForMixin, + MarkdownMixin, + ExtractionMixin, + AskMixin, + FillFieldsMixin, + ClickMixin, + KeyboardMixin, + GetElementMixin, + ABC, +): + """ + Dendrite is a class that manages a browser instance using Playwright, allowing + interactions with web pages using natural language. + + This class handles initialization with API keys for Dendrite, OpenAI, and Anthropic, manages browser + contexts, and provides methods for navigation, authentication, and other browser-related tasks. + + Attributes: + id (UUID): The unique identifier for the Dendrite instance. + auth_data (Optional[AuthSession]): The authentication session data for the browser. + dendrite_api_key (str): The API key for Dendrite, used for interactions with the Dendrite API. + playwright_options (dict): Options for configuring the Playwright browser instance. + playwright (Optional[Playwright]): The Playwright instance managing the browser. + browser_context (Optional[BrowserContext]): The current browser context, which may include cookies and other session data. + active_page_manager (Optional[PageManager]): The manager responsible for handling active pages within the browser context. + user_id (Optional[str]): The user ID associated with the browser session. + browser_api_client (BrowserAPIClient): The API client used for communicating with the Dendrite API. + api_config (APIConfig): The configuration for the language models, including API keys for OpenAI and Anthropic. + + Raises: + Exception: If any of the required API keys (Dendrite, OpenAI, Anthropic) are not provided or found in the environment variables. + """ + + def __init__( + self, + playwright_options: Any = {"headless": False, "args": STEALTH_ARGS}, + remote_config: Optional[Providers] = None, + config: Optional[Config] = None, + auth: Optional[Union[List[str], str]] = None, + ): + """ + Initialize Dendrite with optional domain authentication. + + Args: + playwright_options: Options for configuring Playwright + remote_config: Remote browser provider configuration + config: Configuration object + auth: List of domains or single domain to load authentication state for + """ + self._impl = self._get_impl(remote_config) + self._playwright_options = playwright_options + self._config = config or Config() + auth_url = [auth] if isinstance(auth, str) else auth or [] + self._auth_domains = [get_domain_w_suffix(url) for url in auth_url] + self._id = uuid4().hex + self._active_page_manager: Optional[PageManager] = None + self._user_id: Optional[str] = None + self._upload_handler = EventSync(event_type=FileChooser) + self._download_handler = EventSync(event_type=Download) + self.closed = False + self._browser_api_client: LogicEngine = LogicEngine(self._config) + + @property + def pages(self) -> List[Page]: + """ + Retrieves the list of active pages managed by the PageManager. + + Returns: + List[Page]: The list of active pages. + """ + if self._active_page_manager: + return self._active_page_manager.pages + else: + raise BrowserNotLaunchedError() + + def _get_page(self) -> Page: + active_page = self.get_active_page() + return active_page + + @property + def logic_engine(self) -> LogicEngine: + return self._browser_api_client + + @property + def dendrite_browser(self) -> "Dendrite": + return self + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def _get_impl(self, remote_provider: Optional[Providers]) -> BrowserProtocol: + return get_impl(remote_provider) + + def get_active_page(self) -> Page: + """ + Retrieves the currently active page managed by the PageManager. + + Returns: + Page: The active page object. + + Raises: + Exception: If there is an issue retrieving the active page. + """ + active_page_manager = self._get_active_page_manager() + return active_page_manager.get_active_page() + + def new_tab( + self, url: str, timeout: Optional[float] = 15000, expected_page: str = "" + ) -> Page: + """ + Opens a new tab and navigates to the specified URL. + + Args: + url (str): The URL to navigate to. + timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. + expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. + + Returns: + Page: The page object after navigation. + + Raises: + Exception: If there is an error during navigation or if the expected page type is not found. + """ + return self.goto( + url, new_tab=True, timeout=timeout, expected_page=expected_page + ) + + def goto( + self, + url: str, + new_tab: bool = False, + timeout: Optional[float] = 15000, + expected_page: str = "", + ) -> Page: + """ + Navigates to the specified URL, optionally in a new tab + + Args: + url (str): The URL to navigate to. + new_tab (bool, optional): Whether to open the URL in a new tab. Defaults to False. + timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. + expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. + + Returns: + Page: The page object after navigation. + + Raises: + Exception: If there is an error during navigation or if the expected page type is not found. + """ + if not re.match("^\\w+://", url): + url = f"https://{url}" + active_page_manager = self._get_active_page_manager() + if new_tab: + active_page = active_page_manager.new_page() + else: + active_page = active_page_manager.get_active_page() + try: + logger.info(f"Going to {url}") + active_page.playwright_page.goto(url, timeout=timeout) + except TimeoutError: + logger.debug("Timeout when loading page but continuing anyways.") + except Exception as e: + logger.debug(f"Exception when loading page but continuing anyways. {e}") + if expected_page != "": + try: + prompt = f"We are checking if we have arrived on the expected type of page. If it is apparent that we have arrived on the wrong page, output an error. Here is the description: '{expected_page}'" + active_page.ask(prompt, bool) + except DendriteException as e: + raise IncorrectOutcomeError(f"Incorrect navigation, reason: {e}") + return active_page + + def scroll_to_bottom( + self, + timeout: float = 30000, + scroll_increment: int = 1000, + no_progress_limit: int = 3, + ): + """ + Scrolls to the bottom of the current page. + + Returns: + None + """ + active_page = self.get_active_page() + active_page.scroll_to_bottom( + timeout=timeout, + scroll_increment=scroll_increment, + no_progress_limit=no_progress_limit, + ) + + def _launch(self): + """ + Launches the Playwright instance and sets up the browser context and page manager. + + This method initializes the Playwright instance, creates a browser context, and sets up the PageManager. + It also applies any authentication data if available. + + Returns: + Tuple[Browser, BrowserContext, PageManager]: The launched browser, context, and page manager. + + Raises: + Exception: If there is an issue launching the browser or setting up the context. + """ + os.environ["PW_TEST_SCREENSHOT_NO_FONTS_READY"] = "1" + self._playwright = sync_playwright().start() + storage_states = [] + for domain in self._auth_domains: + state = self._get_domain_storage_state(domain) + if state: + storage_states.append(state) + browser = self._impl.start_browser(self._playwright, self._playwright_options) + if storage_states: + merged_state = self._merge_storage_states(storage_states) + self.browser_context = browser.new_context(storage_state=merged_state) + else: + self.browser_context = ( + browser.contexts[0] + if len(browser.contexts) > 0 + else browser.new_context() + ) + self._active_page_manager = PageManager(self, self.browser_context) + self._impl.configure_context(self) + return (browser, self.browser_context, self._active_page_manager) + + def add_cookies(self, cookies): + """ + Adds cookies to the current browser context. + + Args: + cookies (List[Dict[str, Any]]): A list of cookies to be added to the browser context. + + Raises: + Exception: If the browser context is not initialized. + """ + if not self.browser_context: + raise DendriteException("Browser context not initialized") + self.browser_context.add_cookies(cookies) + + def close(self): + """ + Closes the browser and updates storage states for authenticated domains before cleanup. + + This method updates the storage states for authenticated domains, stops the Playwright + instance, and closes the browser context. + + Returns: + None + + Raises: + Exception: If there is an issue closing the browser or updating session data. + """ + self.closed = True + try: + if self.browser_context and self._auth_domains: + for domain in self._auth_domains: + self.save_auth(domain) + self._impl.stop_session() + self.browser_context.close() + except Error: + pass + try: + if self._playwright: + self._playwright.stop() + except (AttributeError, Exception): + pass + + def _is_launched(self): + """ + Checks whether the browser context has been launched. + + Returns: + bool: True if the browser context is launched, False otherwise. + """ + return self.browser_context is not None + + def _get_active_page_manager(self) -> PageManager: + """ + Retrieves the active PageManager instance, launching the browser if necessary. + + Returns: + PageManager: The active PageManager instance. + + Raises: + Exception: If there is an issue launching the browser or retrieving the PageManager. + """ + if not self._active_page_manager: + (_, _, active_page_manager) = self._launch() + return active_page_manager + return self._active_page_manager + + def get_download(self, timeout: float) -> Download: + """ + Retrieves the download event from the browser. + + Returns: + Download: The download event. + + Raises: + Exception: If there is an issue retrieving the download event. + """ + active_page = self.get_active_page() + pw_page = active_page.playwright_page + return self._get_download(pw_page, timeout) + + def _get_download(self, pw_page: PlaywrightPage, timeout: float) -> Download: + """ + Retrieves the download event from the browser. + + Returns: + Download: The download event. + + Raises: + Exception: If there is an issue retrieving the download event. + """ + return self._download_handler.get_data(pw_page, timeout=timeout) + + def upload_files( + self, + files: Union[ + str, + pathlib.Path, + FilePayload, + Sequence[Union[str, pathlib.Path]], + Sequence[FilePayload], + ], + timeout: float = 30000, + ) -> None: + """ + Uploads files to the active page using a file chooser. + + Args: + files (Union[str, pathlib.Path, FilePayload, Sequence[Union[str, pathlib.Path]], Sequence[FilePayload]]): The file(s) to be uploaded. + This can be a file path, a `FilePayload` object, or a sequence of file paths or `FilePayload` objects. + timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the file chooser to be ready. Defaults to 30. + + Returns: + None + """ + page = self.get_active_page() + file_chooser = self._get_filechooser(page.playwright_page, timeout) + file_chooser.set_files(files) + + def _get_filechooser( + self, pw_page: PlaywrightPage, timeout: float = 30000 + ) -> FileChooser: + """ + Uploads files to the browser. + + Args: + timeout (float): The maximum time to wait for the file chooser dialog. Defaults to 30000 milliseconds. + + Returns: + FileChooser: The file chooser dialog. + + Raises: + Exception: If there is an issue uploading files. + """ + return self._upload_handler.get_data(pw_page, timeout=timeout) + + def save_auth(self, url: str) -> None: + """ + Save authentication state for a specific domain. + + Args: + domain (str): Domain to save authentication for (e.g., "github.com") + """ + if not self.browser_context: + raise DendriteException("Browser context not initialized") + domain = get_domain_w_suffix(url) + storage_state = self.browser_context.storage_state() + filtered_state = { + "origins": [ + origin + for origin in storage_state.get("origins", []) + if domain in origin.get("origin", "") + ], + "cookies": [ + cookie + for cookie in storage_state.get("cookies", []) + if domain in cookie.get("domain", "") + ], + } + self._config.storage_cache.set( + {"domain": domain}, StorageState(**filtered_state) + ) + + def setup_auth( + self, + url: str, + message: str = "Please log in to the website. Once done, press Enter to continue...", + ) -> None: + """ + Set up authentication for a specific URL. + + Args: + url (str): URL to navigate to for login + message (str): Message to show while waiting for user input + """ + domain = get_domain_w_suffix(url) + try: + self._playwright = sync_playwright().start() + browser = self._impl.start_browser( + self._playwright, {**self._playwright_options, "headless": False} + ) + self.browser_context = browser.new_context() + self._active_page_manager = PageManager(self, self.browser_context) + self.goto(url) + print(message) + input() + self.save_auth(domain) + finally: + self.close() + + def _get_domain_storage_state(self, domain: str) -> Optional[StorageState]: + """Get storage state for a specific domain""" + return self._config.storage_cache.get({"domain": domain}) + + def _merge_storage_states(self, states: List[StorageState]) -> StorageState: + """Merge multiple storage states into one""" + merged = {"origins": [], "cookies": []} + seen_origins = set() + seen_cookies = set() + for state in states: + for origin in state.get("origins", []): + origin_key = origin.get("origin", "") + if origin_key not in seen_origins: + merged["origins"].append(origin) + seen_origins.add(origin_key) + for cookie in state.get("cookies", []): + cookie_key = ( + f"{cookie.get('name')}:{cookie.get('domain')}:{cookie.get('path')}" + ) + if cookie_key not in seen_cookies: + merged["cookies"].append(cookie) + seen_cookies.add(cookie_key) + return StorageState(**merged) diff --git a/dendrite/browser/sync_api/dendrite_element.py b/dendrite/browser/sync_api/dendrite_element.py new file mode 100644 index 0000000..2ef67a6 --- /dev/null +++ b/dendrite/browser/sync_api/dendrite_element.py @@ -0,0 +1,237 @@ +from __future__ import annotations +import time +import base64 +import functools +import time +from typing import TYPE_CHECKING, Optional +from loguru import logger +from playwright.sync_api import Locator +from dendrite.browser._common._exceptions.dendrite_exception import ( + IncorrectOutcomeError, +) +from dendrite.logic import LogicEngine + +if TYPE_CHECKING: + from .dendrite_browser import Dendrite +from dendrite.models.dto.make_interaction_dto import VerifyActionDTO +from dendrite.models.response.interaction_response import InteractionResponse +from .manager.navigation_tracker import NavigationTracker +from .types import Interaction + + +def perform_action(interaction_type: Interaction): + """ + Decorator for performing actions on DendriteElements. + + This decorator wraps methods of Element to handle interactions, + expected outcomes, and error handling. + + Args: + interaction_type (Interaction): The type of interaction being performed. + + Returns: + function: The decorated function. + """ + + def decorator(func): + + @functools.wraps(func) + def wrapper(self: Element, *args, **kwargs) -> InteractionResponse: + expected_outcome: Optional[str] = kwargs.pop("expected_outcome", None) + if not expected_outcome: + func(self, *args, **kwargs) + return InteractionResponse(status="success", message="") + page_before = self._dendrite_browser.get_active_page() + page_before_info = page_before.get_page_information() + soup = page_before._get_previous_soup() + screenshot_before = page_before_info.screenshot_base64 + tag_name = soup.find(attrs={"d-id": self.dendrite_id}) + func(self, *args, expected_outcome=expected_outcome, **kwargs) + self._wait_for_page_changes(page_before.url) + page_after = self._dendrite_browser.get_active_page() + screenshot_after = page_after.screenshot_manager.take_full_page_screenshot() + dto = VerifyActionDTO( + url=page_before.url, + dendrite_id=self.dendrite_id, + interaction_type=interaction_type, + expected_outcome=expected_outcome, + screenshot_before=screenshot_before, + screenshot_after=screenshot_after, + tag_name=str(tag_name), + ) + res = self._browser_api_client.verify_action(dto) + if res.status == "failed": + raise IncorrectOutcomeError( + message=res.message, screenshot_base64=screenshot_after + ) + return res + + return wrapper + + return decorator + + +class Element: + """ + Represents an element in the Dendrite browser environment. Wraps a Playwright Locator. + + This class provides methods for interacting with and manipulating + elements in the browser. + """ + + def __init__( + self, + dendrite_id: str, + locator: Locator, + dendrite_browser: Dendrite, + browser_api_client: LogicEngine, + ): + """ + Initialize a Element. + + Args: + dendrite_id (str): The dendrite_id identifier for this element. + locator (Locator): The Playwright locator for this element. + dendrite_browser (Dendrite): The browser instance. + """ + self.dendrite_id = dendrite_id + self.locator = locator + self._dendrite_browser = dendrite_browser + self._browser_api_client = browser_api_client + + def outer_html(self): + return self.locator.evaluate("(element) => element.outerHTML") + + def screenshot(self) -> str: + """ + Take a screenshot of the element and return it as a base64-encoded string. + + Returns: + str: A base64-encoded string of the JPEG image. + Returns an empty string if the screenshot fails. + """ + image_data = self.locator.screenshot(type="jpeg", timeout=20000) + if image_data is None: + return "" + return base64.b64encode(image_data).decode() + + @perform_action("click") + def click( + self, + expected_outcome: Optional[str] = None, + wait_for_navigation: bool = True, + *args, + **kwargs, + ) -> InteractionResponse: + """ + Click the element. + + Args: + expected_outcome (Optional[str]): The expected outcome of the click action. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + InteractionResponse: The response from the interaction. + """ + timeout = kwargs.pop("timeout", 2000) + force = kwargs.pop("force", False) + page = self._dendrite_browser.get_active_page() + navigation_tracker = NavigationTracker(page) + navigation_tracker.start_nav_tracking() + try: + self.locator.click(*args, timeout=timeout, force=force, **kwargs) + except Exception as e: + try: + self.locator.click(*args, timeout=2000, force=True, **kwargs) + except Exception as e: + self.locator.dispatch_event("click", timeout=2000) + if wait_for_navigation: + has_navigated = navigation_tracker.has_navigated_since_start() + if has_navigated: + try: + start_time = time.time() + page.playwright_page.wait_for_load_state("load", timeout=2000) + wait_duration = time.time() - start_time + except Exception as e: + pass + return InteractionResponse(status="success", message="") + + @perform_action("fill") + def fill( + self, value: str, expected_outcome: Optional[str] = None, *args, **kwargs + ) -> InteractionResponse: + """ + Fill the element with a value. If the element itself is not fillable, + it attempts to find and fill a fillable child element. + + Args: + value (str): The value to fill the element with. + expected_outcome (Optional[str]): The expected outcome of the fill action. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + InteractionResponse: The response from the interaction. + """ + timeout = kwargs.pop("timeout", 2000) + try: + self.locator.fill(value, *args, timeout=timeout, **kwargs) + except Exception as e: + fillable_child = self.locator.locator( + 'input, textarea, [contenteditable="true"]' + ).first + fillable_child.fill(value, *args, timeout=timeout, **kwargs) + return InteractionResponse(status="success", message="") + + @perform_action("hover") + def hover( + self, expected_outcome: Optional[str] = None, *args, **kwargs + ) -> InteractionResponse: + """ + Hover over the element. + All additional arguments are passed to the Playwright fill method. + + Args: + expected_outcome (Optional[str]): The expected outcome of the hover action. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + InteractionResponse: The response from the interaction. + """ + timeout = kwargs.pop("timeout", 2000) + self.locator.hover(*args, timeout=timeout, **kwargs) + return InteractionResponse(status="success", message="") + + def focus(self): + """ + Focus on the element. + """ + self.locator.focus() + + def highlight(self): + """ + Highlights the element. This is a convenience method for debugging purposes. + """ + self.locator.highlight() + + def _wait_for_page_changes(self, old_url: str, timeout: float = 2000): + """ + Wait for page changes after an action. + + Args: + old_url (str): The URL before the action. + timeout (float): The maximum time (in milliseconds) to wait for changes. + + Returns: + bool: True if the page changed, False otherwise. + """ + timeout_in_seconds = timeout / 1000 + start_time = time.time() + while time.time() - start_time <= timeout_in_seconds: + page = self._dendrite_browser.get_active_page() + if page.url != old_url: + return True + time.sleep(0.1) + return False diff --git a/dendrite/browser/sync_api/dendrite_page.py b/dendrite/browser/sync_api/dendrite_page.py new file mode 100644 index 0000000..d6f2d01 --- /dev/null +++ b/dendrite/browser/sync_api/dendrite_page.py @@ -0,0 +1,377 @@ +import time +import pathlib +import re +import time +from typing import TYPE_CHECKING, Any, List, Literal, Optional, Sequence, Union +from bs4 import BeautifulSoup, Tag +from loguru import logger +from playwright.sync_api import Download, FilePayload, FrameLocator, Keyboard +from dendrite.logic import LogicEngine +from dendrite.models.page_information import PageInformation +from .dendrite_element import Element +from .js import GENERATE_DENDRITE_IDS_SCRIPT +from .mixin.ask import AskMixin +from .mixin.click import ClickMixin +from .mixin.extract import ExtractionMixin +from .mixin.fill_fields import FillFieldsMixin +from .mixin.get_element import GetElementMixin +from .mixin.keyboard import KeyboardMixin +from .mixin.markdown import MarkdownMixin +from .mixin.wait_for import WaitForMixin +from .types import PlaywrightPage + +if TYPE_CHECKING: + from .dendrite_browser import Dendrite +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from ._utils import expand_iframes +from .manager.screenshot_manager import ScreenshotManager + + +class Page( + MarkdownMixin, + ExtractionMixin, + WaitForMixin, + AskMixin, + FillFieldsMixin, + ClickMixin, + KeyboardMixin, + GetElementMixin, +): + """ + Represents a page in the Dendrite browser environment. + + This class provides methods for interacting with and manipulating + pages in the browser. + """ + + def __init__( + self, + page: PlaywrightPage, + dendrite_browser: "Dendrite", + browser_api_client: LogicEngine, + ): + self.playwright_page = page + self.screenshot_manager = ScreenshotManager(page) + self._browser_api_client = browser_api_client + self._last_main_frame_url = page.url + self._last_frame_navigated_timestamp = time.time() + self._dendrite_browser = dendrite_browser + self.playwright_page.on("framenavigated", self._on_frame_navigated) + + def _on_frame_navigated(self, frame): + if frame is self.playwright_page.main_frame: + self._last_main_frame_url = frame.url + self._last_frame_navigated_timestamp = time.time() + + @property + def dendrite_browser(self) -> "Dendrite": + return self._dendrite_browser + + @property + def url(self): + """ + Get the current URL of the page. + + Returns: + str: The current URL. + """ + return self.playwright_page.url + + @property + def keyboard(self) -> Keyboard: + """ + Get the keyboard object for the page. + + Returns: + Keyboard: The Playwright Keyboard object. + """ + return self.playwright_page.keyboard + + def _get_page(self) -> "Page": + return self + + @property + def logic_engine(self) -> LogicEngine: + return self._browser_api_client + + def goto( + self, + url: str, + timeout: Optional[float] = 30000, + wait_until: Optional[ + Literal["commit", "domcontentloaded", "load", "networkidle"] + ] = "load", + ) -> None: + """ + Navigate to a URL. + + Args: + url (str): The URL to navigate to. If no protocol is specified, 'https://' will be added. + timeout (Optional[float]): Maximum navigation time in milliseconds. + wait_until (Optional[Literal["commit", "domcontentloaded", "load", "networkidle"]]): + When to consider navigation succeeded. + """ + if not re.match("^\\w+://", url): + url = f"https://{url}" + self.playwright_page.goto(url, timeout=timeout, wait_until=wait_until) + + def get_download(self, timeout: float = 30000) -> Download: + """ + Retrieves the download event associated with. + + Args: + timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the download to complete. Defaults to 30. + + Returns: + The downloaded file data. + """ + return self.dendrite_browser._get_download(self.playwright_page, timeout) + + def _get_context(self, element: Any) -> Union[PlaywrightPage, FrameLocator]: + """ + Gets the correct context to be able to interact with an element on a different frame. + + e.g. if the element is inside an iframe, + the context will be the frame locator for that iframe. + + Args: + element (Any): The element to get the context for. + + Returns: + Union[Page, FrameLocator]: The context for the element. + """ + context = self.playwright_page + if isinstance(element, Tag): + full_path = element.get("iframe-path") + if full_path: + full_path = full_path[0] if isinstance(full_path, list) else full_path + for path in full_path.split("|"): + context = context.frame_locator(f"xpath=//iframe[@d-id='{path}']") + return context + + def scroll_to_bottom( + self, + timeout: float = 30000, + scroll_increment: int = 1000, + no_progress_limit: int = 3, + ) -> None: + """ + Scrolls to the bottom of the page until no more progress is made or a timeout occurs. + + Args: + timeout (float, optional): The maximum amount of time (in milliseconds) to continue scrolling. Defaults to 30000. + scroll_increment (int, optional): The number of pixels to scroll in each step. Defaults to 1000. + no_progress_limit (int, optional): The number of consecutive attempts with no progress before stopping. Defaults to 3. + + Returns: + None + """ + start_time = time.time() + last_scroll_position = 0 + no_progress_count = 0 + while True: + current_scroll_position = self.playwright_page.evaluate("window.scrollY") + scroll_height = self.playwright_page.evaluate("document.body.scrollHeight") + self.playwright_page.evaluate( + f"window.scrollTo(0, {current_scroll_position + scroll_increment})" + ) + if ( + self.playwright_page.viewport_size + and current_scroll_position + + self.playwright_page.viewport_size["height"] + >= scroll_height + ): + break + if current_scroll_position > last_scroll_position: + no_progress_count = 0 + else: + no_progress_count += 1 + if no_progress_count >= no_progress_limit: + break + if time.time() - start_time > timeout * 0.001: + break + last_scroll_position = current_scroll_position + time.sleep(0.1) + + def close(self) -> None: + """ + Closes the current page. + + Returns: + None + """ + self.playwright_page.close() + + def get_page_information(self, include_screenshot: bool = True) -> PageInformation: + """ + Retrieves information about the current page, including the URL, raw HTML, and a screenshot. + + Returns: + PageInformation: An object containing the page's URL, raw HTML, and a screenshot in base64 format. + """ + if include_screenshot: + base64 = self.screenshot_manager.take_full_page_screenshot() + else: + base64 = "No screenshot available" + soup = self._get_soup() + return PageInformation( + url=self.playwright_page.url, + raw_html=str(soup), + screenshot_base64=base64, + time_since_frame_navigated=self.get_time_since_last_frame_navigated(), + ) + + def _generate_dendrite_ids(self): + """ + Attempts to generate Dendrite IDs in the DOM by executing a script. + + This method will attempt to generate the Dendrite IDs up to 3 times. If all attempts fail, + an exception is raised. + + Raises: + Exception: If the Dendrite IDs could not be generated after 3 attempts. + """ + tries = 0 + while tries < 3: + try: + self.playwright_page.evaluate(GENERATE_DENDRITE_IDS_SCRIPT) + return + except Exception as e: + self.playwright_page.wait_for_load_state(state="load", timeout=3000) + logger.exception( + f"Failed to generate dendrite IDs: {e}, attempt {tries + 1}/3" + ) + tries += 1 + raise DendriteException("Failed to add d-ids to DOM.") + + def scroll_through_entire_page(self) -> None: + """ + Scrolls through the entire page. + + Returns: + None + """ + self.scroll_to_bottom() + + def upload_files( + self, + files: Union[ + str, + pathlib.Path, + FilePayload, + Sequence[Union[str, pathlib.Path]], + Sequence[FilePayload], + ], + timeout: float = 30000, + ) -> None: + """ + Uploads files to the page using a file chooser. + + Args: + files (Union[str, pathlib.Path, FilePayload, Sequence[Union[str, pathlib.Path]], Sequence[FilePayload]]): The file(s) to be uploaded. + This can be a file path, a `FilePayload` object, or a sequence of file paths or `FilePayload` objects. + timeout (float, optional): The maximum amount of time (in milliseconds) to wait for the file chooser to be ready. Defaults to 30. + + Returns: + None + """ + file_chooser = self.dendrite_browser._get_filechooser( + self.playwright_page, timeout + ) + file_chooser.set_files(files) + + def get_content(self): + """ + Retrieves the content of the current page. + + Returns: + str: The HTML content of the current page. + """ + return self.playwright_page.content() + + def _get_soup(self) -> BeautifulSoup: + """ + Retrieves the page source as a BeautifulSoup object, with an option to exclude hidden elements. + Generates Dendrite IDs in the DOM and expands iframes. + + Returns: + BeautifulSoup: The parsed HTML of the current page. + """ + self._generate_dendrite_ids() + page_source = self.playwright_page.content() + soup = BeautifulSoup(page_source, "lxml") + self._expand_iframes(soup) + self._previous_soup = soup + return soup + + def _get_previous_soup(self) -> BeautifulSoup: + """ + Retrieves the page source generated by the latest _get_soup() call as a Beautiful soup object. If it hasn't been called yet, it will call it. + """ + if self._previous_soup is None: + return self._get_soup() + return self._previous_soup + + def _expand_iframes(self, page_source: BeautifulSoup): + """ + Expands iframes in the given page source to make their content accessible. + + Args: + page_source (BeautifulSoup): The parsed HTML content of the page. + + Returns: + None + """ + expand_iframes(self.playwright_page, page_source) + + def _get_all_elements_from_selector(self, selector: str) -> List[Element]: + dendrite_elements: List[Element] = [] + soup = self._get_soup() + elements = soup.select(selector) + for element in elements: + frame = self._get_context(element) + d_id = element.get("d-id", "") + locator = frame.locator(f"xpath=//*[@d-id='{d_id}']") + if not d_id: + continue + if isinstance(d_id, list): + d_id = d_id[0] + dendrite_elements.append( + Element(d_id, locator, self.dendrite_browser, self._browser_api_client) + ) + return dendrite_elements + + def _dump_html(self, path: str) -> None: + """ + Saves the current page's HTML content to a file. + + Args: + path (str): The file path where the HTML content should be saved. + + Returns: + None + """ + with open(path, "w") as f: + f.write(self.playwright_page.content()) + + def get_time_since_last_frame_navigated(self) -> float: + """ + Get the time elapsed since the last URL change. + + Returns: + float: The number of seconds elapsed since the last URL change. + """ + return time.time() - self._last_frame_navigated_timestamp + + def check_if_renavigated(self, initial_url: str, wait_time: float = 0.1) -> bool: + """ + Waits for a short period and checks if a main frame navigation has occurred. + + Args: + wait_time (float): The time to wait in seconds. Defaults to 0.1 seconds. + + Returns: + bool: True if a main frame navigation occurred, False otherwise. + """ + time.sleep(wait_time) + return self._last_main_frame_url != initial_url diff --git a/dendrite/browser/sync_api/js/__init__.py b/dendrite/browser/sync_api/js/__init__.py new file mode 100644 index 0000000..ccaf080 --- /dev/null +++ b/dendrite/browser/sync_api/js/__init__.py @@ -0,0 +1,11 @@ +from pathlib import Path + + +def load_script(filename: str) -> str: + current_dir = Path(__file__).parent + file_path = current_dir / filename + return file_path.read_text(encoding="utf-8") + + +GENERATE_DENDRITE_IDS_SCRIPT = load_script("generateDendriteIDs.js") +GENERATE_DENDRITE_IDS_IFRAME_SCRIPT = load_script("generateDendriteIDsIframe.js") diff --git a/dendrite/browser/sync_api/js/eventListenerPatch.js b/dendrite/browser/sync_api/js/eventListenerPatch.js new file mode 100644 index 0000000..7f03d55 --- /dev/null +++ b/dendrite/browser/sync_api/js/eventListenerPatch.js @@ -0,0 +1,90 @@ +// Save the original methods before redefining them +EventTarget.prototype._originalAddEventListener = EventTarget.prototype.addEventListener; +EventTarget.prototype._originalRemoveEventListener = EventTarget.prototype.removeEventListener; + +// Redefine the addEventListener method +EventTarget.prototype.addEventListener = function(event, listener, options = false) { + // Initialize the eventListenerList if it doesn't exist + if (!this.eventListenerList) { + this.eventListenerList = {}; + } + // Initialize the event list for the specific event if it doesn't exist + if (!this.eventListenerList[event]) { + this.eventListenerList[event] = []; + } + // Add the event listener details to the event list + this.eventListenerList[event].push({ listener, options, outerHTML: this.outerHTML }); + + // Call the original addEventListener method + this._originalAddEventListener(event, listener, options); +}; + +// Redefine the removeEventListener method +EventTarget.prototype.removeEventListener = function(event, listener, options = false) { + // Remove the event listener details from the event list + if (this.eventListenerList && this.eventListenerList[event]) { + this.eventListenerList[event] = this.eventListenerList[event].filter( + item => item.listener !== listener + ); + } + + // Call the original removeEventListener method + this._originalRemoveEventListener( event, listener, options); +}; + +// Get event listeners for a specific event type or all events if not specified +EventTarget.prototype._getEventListeners = function(eventType) { + if (!this.eventListenerList) { + this.eventListenerList = {}; + } + + const eventsToCheck = ['click', 'dblclick', 'mousedown', 'mouseup', 'mouseover', 'mouseout', 'mousemove', 'keydown', 'keyup', 'keypress']; + + eventsToCheck.forEach(type => { + if (!eventType || eventType === type) { + if (this[`on${type}`]) { + if (!this.eventListenerList[type]) { + this.eventListenerList[type] = []; + } + this.eventListenerList[type].push({ listener: this[`on${type}`], inline: true }); + } + } + }); + + return eventType === undefined ? this.eventListenerList : this.eventListenerList[eventType]; +}; + +// Utility to show events +function _showEvents(events) { + let result = ''; + for (let event in events) { + result += `${event} ----------------> ${events[event].length}\n`; + for (let listenerObj of events[event]) { + result += `${listenerObj.listener.toString()}\n`; + } + } + return result; +} + +// Extend EventTarget prototype with utility methods +EventTarget.prototype.on = function(event, callback, options) { + this.addEventListener(event, callback, options); + return this; +}; + +EventTarget.prototype.off = function(event, callback, options) { + this.removeEventListener(event, callback, options); + return this; +}; + +EventTarget.prototype.emit = function(event, args = null) { + this.dispatchEvent(new CustomEvent(event, { detail: args })); + return this; +}; + +// Make these methods non-enumerable +Object.defineProperties(EventTarget.prototype, { + on: { enumerable: false }, + off: { enumerable: false }, + emit: { enumerable: false } +}); diff --git a/dendrite/browser/sync_api/js/generateDendriteIDs.js b/dendrite/browser/sync_api/js/generateDendriteIDs.js new file mode 100644 index 0000000..0ae5f61 --- /dev/null +++ b/dendrite/browser/sync_api/js/generateDendriteIDs.js @@ -0,0 +1,88 @@ +var hashCode = (str) => { + var hash = 0, i, chr; + if (str.length === 0) return hash; + for (i = 0; i < str.length; i++) { + chr = str.charCodeAt(i); + hash = ((hash << 5) - hash) + chr; + hash |= 0; // Convert to 32bit integer + } + return hash; +} + + +const getElementIndex = (element) => { + let index = 1; + let sibling = element.previousElementSibling; + + while (sibling) { + if (sibling.localName === element.localName) { + index++; + } + sibling = sibling.previousElementSibling; + } + + return index; +}; + + +const segs = function elmSegs(elm) { + if (!elm || elm.nodeType !== 1) return ['']; + if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; + const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; + let index = getElementIndex(elm); + + return [...elmSegs(elm.parentNode), `${localName}[${index}]`]; +}; + +var getXPathForElement = (element) => { + return segs(element).join('/'); +} + +// Create a Map to store used hashes and their counters +const usedHashes = new Map(); + +var markHidden = (hidden_element) => { + // Mark the hidden element itself + hidden_element.setAttribute('data-hidden', 'true'); + +} + +document.querySelectorAll('*').forEach((element, index) => { + try { + + const xpath = getXPathForElement(element); + const hash = hashCode(xpath); + const baseId = hash.toString(36); + + // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; + const isHidden = !element.checkVisibility(); + // computedStyle.width === '0px' || + // computedStyle.height === '0px'; + + if (isHidden) { + markHidden(element); + }else{ + element.removeAttribute("data-hidden") // in case we hid it in a previous call + } + + let uniqueId = baseId; + let counter = 0; + + // Check if this hash has been used before + while (usedHashes.has(uniqueId)) { + // If it has, increment the counter and create a new uniqueId + counter++; + uniqueId = `${baseId}_${counter}`; + } + + // Add the uniqueId to the usedHashes Map + usedHashes.set(uniqueId, true); + element.setAttribute('d-id', uniqueId); + } catch (error) { + // Fallback: use a hash of the tag name and index + const fallbackId = hashCode(`${element.tagName}_${index}`).toString(36); + console.error('Error processing element, using fallback:',fallbackId, element, error); + + element.setAttribute('d-id', `fallback_${fallbackId}`); + } +}); \ No newline at end of file diff --git a/dendrite/browser/sync_api/js/generateDendriteIDsIframe.js b/dendrite/browser/sync_api/js/generateDendriteIDsIframe.js new file mode 100644 index 0000000..4f59cef --- /dev/null +++ b/dendrite/browser/sync_api/js/generateDendriteIDsIframe.js @@ -0,0 +1,93 @@ +({frame_path}) => { + var hashCode = (str) => { + var hash = 0, i, chr; + if (str.length === 0) return hash; + for (i = 0; i < str.length; i++) { + chr = str.charCodeAt(i); + hash = ((hash << 5) - hash) + chr; + hash |= 0; // Convert to 32bit integer + } + return hash; + } + + const getElementIndex = (element) => { + let index = 1; + let sibling = element.previousElementSibling; + + while (sibling) { + if (sibling.localName === element.localName) { + index++; + } + sibling = sibling.previousElementSibling; + } + + return index; + }; + + + const segs = function elmSegs(elm) { + if (!elm || elm.nodeType !== 1) return ['']; + if (elm.id && document.getElementById(elm.id) === elm) return [`id("${elm.id}")`]; + const localName = typeof elm.localName === 'string' ? elm.localName.toLowerCase() : 'unknown'; + let index = getElementIndex(elm); + + return [...elmSegs(elm.parentNode), `${localName}[${index}]`]; + }; + + var getXPathForElement = (element) => { + return segs(element).join('/'); + } + + // Create a Map to store used hashes and their counters + const usedHashes = new Map(); + + var markHidden = (hidden_element) => { + // Mark the hidden element itself + hidden_element.setAttribute('data-hidden', 'true'); + } + + document.querySelectorAll('*').forEach((element, index) => { + try { + + + // const is_marked_hidden = element.getAttribute("data-hidden") === "true"; + const isHidden = !element.checkVisibility(); + // computedStyle.width === '0px' || + // computedStyle.height === '0px'; + + if (isHidden) { + markHidden(element); + }else{ + element.removeAttribute("data-hidden") // in case we hid it in a previous call + } + let xpath = getXPathForElement(element); + if(frame_path){ + element.setAttribute("iframe-path",frame_path) + xpath = frame_path + xpath; + } + const hash = hashCode(xpath); + const baseId = hash.toString(36); + + let uniqueId = baseId; + let counter = 0; + + // Check if this hash has been used before + while (usedHashes.has(uniqueId)) { + // If it has, increment the counter and create a new uniqueId + counter++; + uniqueId = `${baseId}_${counter}`; + } + + // Add the uniqueId to the usedHashes Map + usedHashes.set(uniqueId, true); + element.setAttribute('d-id', uniqueId); + } catch (error) { + // Fallback: use a hash of the tag name and index + const fallbackId = hashCode(`${element.tagName}_${index}`).toString(36); + console.error('Error processing element, using fallback:',fallbackId, element, error); + + element.setAttribute('d-id', `fallback_${fallbackId}`); + } + }); +} + diff --git a/dendrite/browser/sync_api/manager/__init__.py b/dendrite/browser/sync_api/manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/manager/navigation_tracker.py b/dendrite/browser/sync_api/manager/navigation_tracker.py new file mode 100644 index 0000000..d789796 --- /dev/null +++ b/dendrite/browser/sync_api/manager/navigation_tracker.py @@ -0,0 +1,67 @@ +import time +import time +from typing import TYPE_CHECKING, Dict, Optional + +if TYPE_CHECKING: + from ..dendrite_page import Page + + +class NavigationTracker: + + def __init__(self, page: "Page"): + self.playwright_page = page.playwright_page + self._nav_start_timestamp: Optional[float] = None + self.playwright_page.on("framenavigated", self._on_frame_navigated) + self.playwright_page.on("popup", self._on_popup) + self._last_events: Dict[str, Optional[float]] = { + "framenavigated": None, + "popup": None, + } + + def _on_frame_navigated(self, frame): + self._last_events["framenavigated"] = time.time() + if frame is self.playwright_page.main_frame: + self._last_main_frame_url = frame.url + self._last_frame_navigated_timestamp = time.time() + + def _on_popup(self, page): + self._last_events["popup"] = time.time() + + def start_nav_tracking(self): + """Call this just before performing an action that might trigger navigation""" + self._nav_start_timestamp = time.time() + for event in self._last_events: + self._last_events[event] = None + + def get_nav_events_since_start(self): + """ + Returns which events have fired since start_nav_tracking() was called + and how long after the start they occurred + """ + if self._nav_start_timestamp is None: + return "Navigation tracking not started. Call start_nav_tracking() first." + results = {} + for event, timestamp in self._last_events.items(): + if timestamp is not None: + delay = timestamp - self._nav_start_timestamp + results[event] = f"{delay:.3f}s" + else: + results[event] = "not fired" + return results + + def has_navigated_since_start(self): + """Returns True if any navigation event has occurred since start_nav_tracking()""" + if self._nav_start_timestamp is None: + return False + start_time = time.time() + max_wait = 1.0 + while time.time() - start_time < max_wait: + if any( + ( + timestamp is not None and timestamp > self._nav_start_timestamp + for timestamp in self._last_events.values() + ) + ): + return True + time.sleep(0.1) + return False diff --git a/dendrite/browser/sync_api/manager/page_manager.py b/dendrite/browser/sync_api/manager/page_manager.py new file mode 100644 index 0000000..52b5782 --- /dev/null +++ b/dendrite/browser/sync_api/manager/page_manager.py @@ -0,0 +1,82 @@ +from typing import TYPE_CHECKING, Optional +from loguru import logger +from playwright.sync_api import BrowserContext, Download, FileChooser + +if TYPE_CHECKING: + from ..dendrite_browser import Dendrite +from ..dendrite_page import Page +from ..types import PlaywrightPage + + +class PageManager: + + def __init__(self, dendrite_browser, browser_context: BrowserContext): + self.pages: list[Page] = [] + self.active_page: Optional[Page] = None + self.browser_context = browser_context + self.dendrite_browser: Dendrite = dendrite_browser + existing_pages = browser_context.pages + if existing_pages: + for page in existing_pages: + client = self.dendrite_browser.logic_engine + dendrite_page = Page(page, self.dendrite_browser, client) + self.pages.append(dendrite_page) + if self.active_page is None: + self.active_page = dendrite_page + browser_context.on("page", self._page_on_open_handler) + + def new_page(self) -> Page: + new_page = self.browser_context.new_page() + if self.active_page and new_page == self.active_page.playwright_page: + return self.active_page + client = self.dendrite_browser.logic_engine + dendrite_page = Page(new_page, self.dendrite_browser, client) + self.pages.append(dendrite_page) + self.active_page = dendrite_page + return dendrite_page + + def get_active_page(self) -> Page: + if self.active_page is None: + return self.new_page() + return self.active_page + + def _page_on_close_handler(self, page: PlaywrightPage): + if self.browser_context and (not self.dendrite_browser.closed): + copy_pages = self.pages.copy() + is_active_page = False + for dendrite_page in copy_pages: + if dendrite_page.playwright_page == page: + self.pages.remove(dendrite_page) + if dendrite_page == self.active_page: + is_active_page = True + break + for i in reversed(range(len(self.pages))): + try: + self.active_page = self.pages[i] + self.pages[i].playwright_page.bring_to_front() + break + except Exception as e: + logger.warning(f"Error switching to the next page: {e}") + continue + + def _page_on_crash_handler(self, page: PlaywrightPage): + logger.error(f"Page crashed: {page.url}") + page.reload() + + def _page_on_download_handler(self, download: Download): + logger.debug(f"Download started: {download.url}") + self.dendrite_browser._download_handler.set_event(download) + + def _page_on_filechooser_handler(self, file_chooser: FileChooser): + logger.debug("File chooser opened") + self.dendrite_browser._upload_handler.set_event(file_chooser) + + def _page_on_open_handler(self, page: PlaywrightPage): + page.on("close", self._page_on_close_handler) + page.on("crash", self._page_on_crash_handler) + page.on("download", self._page_on_download_handler) + page.on("filechooser", self._page_on_filechooser_handler) + client = self.dendrite_browser.logic_engine + dendrite_page = Page(page, self.dendrite_browser, client) + self.pages.append(dendrite_page) + self.active_page = dendrite_page diff --git a/dendrite/browser/sync_api/manager/screenshot_manager.py b/dendrite/browser/sync_api/manager/screenshot_manager.py new file mode 100644 index 0000000..7f4fd33 --- /dev/null +++ b/dendrite/browser/sync_api/manager/screenshot_manager.py @@ -0,0 +1,50 @@ +import base64 +import os +from uuid import uuid4 +from ..types import PlaywrightPage + + +class ScreenshotManager: + + def __init__(self, page: PlaywrightPage) -> None: + self.screenshot_before: str = "" + self.screenshot_after: str = "" + self.page = page + + def take_full_page_screenshot(self) -> str: + try: + scroll_height = self.page.evaluate( + "\n () => {\n const body = document.body;\n if (!body) {\n return 0; // Return 0 if body is null\n }\n return body.scrollHeight || 0;\n }\n " + ) + if scroll_height > 30000: + print( + f"Page height ({scroll_height}px) exceeds 30000px. Taking viewport screenshot instead." + ) + return self.take_viewport_screenshot() + image_data = self.page.screenshot( + type="jpeg", full_page=True, timeout=10000 + ) + except Exception as e: + print( + f"Full-page screenshot failed: {e}. Falling back to viewport screenshot." + ) + return self.take_viewport_screenshot() + if image_data is None: + return "" + return base64.b64encode(image_data).decode("utf-8") + + def take_viewport_screenshot(self) -> str: + image_data = self.page.screenshot(type="jpeg", timeout=10000) + if image_data is None: + return "" + reduced_base64 = base64.b64encode(image_data).decode("utf-8") + return reduced_base64 + + def store_screenshot(self, name, image_data): + if not name: + name = str(uuid4()) + filepath = os.path.join("test", f"{name}.jpeg") + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "wb") as file: + file.write(image_data) + return filepath diff --git a/dendrite/browser/sync_api/mixin/__init__.py b/dendrite/browser/sync_api/mixin/__init__.py new file mode 100644 index 0000000..046a61c --- /dev/null +++ b/dendrite/browser/sync_api/mixin/__init__.py @@ -0,0 +1,21 @@ +from .ask import AskMixin +from .click import ClickMixin +from .extract import ExtractionMixin +from .fill_fields import FillFieldsMixin +from .get_element import GetElementMixin +from .keyboard import KeyboardMixin +from .markdown import MarkdownMixin +from .screenshot import ScreenshotMixin +from .wait_for import WaitForMixin + +__all__ = [ + "AskMixin", + "ClickMixin", + "ExtractionMixin", + "FillFieldsMixin", + "GetElementMixin", + "KeyboardMixin", + "MarkdownMixin", + "ScreenshotMixin", + "WaitForMixin", +] diff --git a/dendrite/browser/sync_api/mixin/ask.py b/dendrite/browser/sync_api/mixin/ask.py new file mode 100644 index 0000000..57f4a56 --- /dev/null +++ b/dendrite/browser/sync_api/mixin/ask.py @@ -0,0 +1,184 @@ +import time +import time +from typing import Optional, Type, overload +from loguru import logger +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.browser.sync_api._utils import convert_to_type_spec, to_json_schema +from dendrite.models.dto.ask_page_dto import AskPageDTO +from ..protocol.page_protocol import DendritePageProtocol +from ..types import JsonSchema, PydanticModel, TypeSpec + +TIMEOUT_INTERVAL = [150, 450, 1000] + + +class AskMixin(DendritePageProtocol): + + @overload + def ask(self, prompt: str, type_spec: Type[str]) -> str: + """ + Asks a question about the current page and expects a response of type `str`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[str]): The expected return type, which is `str`. + + Returns: + AskPageResponse[str]: The response object containing the result of type `str`. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[bool]) -> bool: + """ + Asks a question about the current page and expects a responseof type `bool`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[bool]): The expected return type, which is `bool`. + + Returns: + AskPageResponse[bool]: The response object containing the result of type `bool`. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[int]) -> int: + """ + Asks a question about the current page and expects a response of type `int`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[int]): The expected return type, which is `int`. + + Returns: + AskPageResponse[int]: The response object containing the result of type `int`. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[float]) -> float: + """ + Asks a question about the current page and expects a response of type `float`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[float]): The expected return type, which is `float`. + + Returns: + AskPageResponse[float]: The response object containing the result of type `float`. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[PydanticModel]) -> PydanticModel: + """ + Asks a question about the current page and expects a response of a custom `PydanticModel`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[PydanticModel]): The expected return type, which is a `PydanticModel`. + + Returns: + AskPageResponse[PydanticModel]: The response object containing the result of the specified Pydantic model type. + """ + + @overload + def ask(self, prompt: str, type_spec: Type[JsonSchema]) -> JsonSchema: + """ + Asks a question about the current page and expects a response conforming to a `JsonSchema`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Type[JsonSchema]): The expected return type, which is a `JsonSchema`. + + Returns: + AskPageResponse[JsonSchema]: The response object containing the result conforming to the specified JSON schema. + """ + + @overload + def ask(self, prompt: str, type_spec: None = None) -> JsonSchema: + """ + Asks a question without specifying a type and expects a response conforming to a default `JsonSchema`. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (None, optional): The expected return type, which is `None` by default. + + Returns: + AskPageResponse[JsonSchema]: The response object containing the result conforming to the default JSON schema. + """ + + def ask( + self, prompt: str, type_spec: Optional[TypeSpec] = None, timeout: int = 15000 + ) -> TypeSpec: + """ + Asks a question and processes the response based on the specified type. + + This method sends a request to ask a question with the specified prompt and processes the response. + If a type specification is provided, the response is converted to the specified type. In case of failure, + a DendriteException is raised with relevant details. + + Args: + prompt (str): The question or prompt to be asked. + type_spec (Optional[TypeSpec], optional): The expected return type, which can be a type or a schema. Defaults to None. + + Returns: + AskPageResponse[Any]: The response object containing the result, converted to the specified type if provided. + + Raises: + DendriteException: If the request fails, the exception includes the failure message and a screenshot. + """ + start_time = time.time() + attempt_start = start_time + attempt = -1 + while True: + attempt += 1 + current_timeout = ( + TIMEOUT_INTERVAL[attempt] + if len(TIMEOUT_INTERVAL) > attempt + else TIMEOUT_INTERVAL[-1] * 1.75 + ) + elapsed_time = time.time() - start_time + remaining_time = timeout * 0.001 - elapsed_time + if remaining_time <= 0: + logger.warning( + f"Timeout reached for '{prompt}' after {attempt + 1} attempts" + ) + break + prev_attempt_time = time.time() - attempt_start + sleep_time = min( + max(current_timeout * 0.001 - prev_attempt_time, 0), remaining_time + ) + logger.debug(f"Waiting for {sleep_time} seconds before retrying") + time.sleep(sleep_time) + attempt_start = time.time() + logger.info(f"Asking '{prompt}' | Attempt {attempt + 1}") + page = self._get_page() + page_information = page.get_page_information() + schema = to_json_schema(type_spec) if type_spec else None + if elapsed_time < 5: + time_prompt = f"This page was loaded {elapsed_time} seconds ago, so it might still be loading. If the page is still loading, return failed status." + else: + time_prompt = "" + entire_prompt = prompt + time_prompt + dto = AskPageDTO( + page_information=page_information, + prompt=entire_prompt, + return_schema=schema, + ) + try: + res = self.logic_engine.ask_page(dto) + logger.debug(f"Got response in {time.time() - attempt_start} seconds") + if res.status == "error": + logger.warning( + f"Error response on attempt {attempt + 1}: {res.return_data}" + ) + continue + converted_res = res.return_data + if type_spec is not None: + converted_res = convert_to_type_spec(type_spec, res.return_data) + return converted_res + except Exception as e: + logger.error(f"Exception occurred on attempt {attempt + 1}: {str(e)}") + if attempt == len(TIMEOUT_INTERVAL) - 1: + raise + raise DendriteException( + message=f"Failed to get response for '{prompt}' after {attempt + 1} attempts", + screenshot_base64=page_information.screenshot_base64, + ) diff --git a/dendrite/browser/sync_api/mixin/click.py b/dendrite/browser/sync_api/mixin/click.py new file mode 100644 index 0000000..2f8461b --- /dev/null +++ b/dendrite/browser/sync_api/mixin/click.py @@ -0,0 +1,56 @@ +from typing import Optional +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.models.response.interaction_response import InteractionResponse +from ..mixin.get_element import GetElementMixin +from ..protocol.page_protocol import DendritePageProtocol + + +class ClickMixin(GetElementMixin, DendritePageProtocol): + + def click( + self, + prompt: str, + expected_outcome: Optional[str] = None, + use_cache: bool = True, + timeout: int = 15000, + force: bool = False, + *args, + **kwargs, + ) -> InteractionResponse: + """ + Clicks an element on the page based on the provided prompt. + + This method combines the functionality of get_element and click, + allowing for a more concise way to interact with elements on the page. + + Args: + prompt (str): The prompt describing the element to be clicked. + expected_outcome (Optional[str]): The expected outcome of the click action. + use_cache (bool, optional): Whether to use cached results for element retrieval. Defaults to True. + timeout (int, optional): The timeout (in milliseconds) for the click operation. Defaults to 15000. + force (bool, optional): Whether to force the click operation. Defaults to False. + *args: Additional positional arguments for the click operation. + **kwargs: Additional keyword arguments for the click operation. + + Returns: + InteractionResponse: The response from the interaction. + + Raises: + DendriteException: If no suitable element is found or if the click operation fails. + """ + augmented_prompt = prompt + "\n\nThe element should be clickable." + element = self.get_element( + augmented_prompt, use_cache=use_cache, timeout=timeout + ) + if not element: + raise DendriteException( + message=f"No element found with the prompt: {prompt}", + screenshot_base64="", + ) + return element.click( + *args, + expected_outcome=expected_outcome, + timeout=timeout, + force=force, + **kwargs, + ) diff --git a/dendrite/browser/sync_api/mixin/extract.py b/dendrite/browser/sync_api/mixin/extract.py new file mode 100644 index 0000000..292fb4f --- /dev/null +++ b/dendrite/browser/sync_api/mixin/extract.py @@ -0,0 +1,277 @@ +import time +import time +from typing import Any, Callable, List, Optional, Type, overload +from loguru import logger +from dendrite.browser.sync_api._utils import convert_to_type_spec, to_json_schema +from dendrite.logic.code.code_session import execute +from dendrite.models.dto.cached_extract_dto import CachedExtractDTO +from dendrite.models.dto.extract_dto import ExtractDTO +from dendrite.models.response.extract_response import ExtractResponse +from dendrite.models.scripts import Script +from ..manager.navigation_tracker import NavigationTracker +from ..protocol.page_protocol import DendritePageProtocol +from ..types import JsonSchema, PydanticModel, TypeSpec + +CACHE_TIMEOUT = 5 + + +class ExtractionMixin(DendritePageProtocol): + """ + Mixin that provides extraction functionality for web pages. + + This mixin provides various `extract` methods that allow extracting + different types of data (e.g., bool, int, float, string, Pydantic models, etc.) + from a web page based on a given prompt. + """ + + @overload + def extract( + self, + prompt: str, + type_spec: Type[bool], + use_cache: bool = True, + timeout: int = 180, + ) -> bool: ... + + @overload + def extract( + self, + prompt: str, + type_spec: Type[int], + use_cache: bool = True, + timeout: int = 180, + ) -> int: ... + + @overload + def extract( + self, + prompt: str, + type_spec: Type[float], + use_cache: bool = True, + timeout: int = 180, + ) -> float: ... + + @overload + def extract( + self, + prompt: str, + type_spec: Type[str], + use_cache: bool = True, + timeout: int = 180, + ) -> str: ... + + @overload + def extract( + self, + prompt: Optional[str], + type_spec: Type[PydanticModel], + use_cache: bool = True, + timeout: int = 180, + ) -> PydanticModel: ... + + @overload + def extract( + self, + prompt: Optional[str], + type_spec: JsonSchema, + use_cache: bool = True, + timeout: int = 180, + ) -> JsonSchema: ... + + @overload + def extract( + self, + prompt: str, + type_spec: None = None, + use_cache: bool = True, + timeout: int = 180, + ) -> Any: ... + + def extract( + self, + prompt: Optional[str], + type_spec: Optional[TypeSpec] = None, + use_cache: bool = True, + timeout: int = 180, + ) -> TypeSpec: + """ + Extract data from a web page based on a prompt and optional type specification. + Args: + prompt (Optional[str]): The prompt to describe the information to extract. + type_spec (Optional[TypeSpec], optional): The type specification for the extracted data. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached scripts before falling back to the + extraction agent for the remaining time that will attempt to generate a new script. Defaults to 15000 (15 seconds). + + Returns: + ExtractResponse: The extracted data wrapped in a ExtractResponse object. + Raises: + TimeoutError: If the extraction process exceeds the specified timeout. + """ + logger.info(f"Starting extraction with prompt: {prompt}") + json_schema = None + if type_spec: + json_schema = to_json_schema(type_spec) + logger.debug(f"Type specification converted to JSON schema: {json_schema}") + if prompt is None: + prompt = "" + start_time = time.time() + page = self._get_page() + navigation_tracker = NavigationTracker(page) + navigation_tracker.start_nav_tracking() + if use_cache: + logger.info("Testing cache") + cached_result = self._try_cached_extraction(prompt, json_schema) + if cached_result: + return convert_and_return_result(cached_result, type_spec) + logger.info( + "Using extraction agent to perform extraction, since no cache was found or failed." + ) + result = self._extract_with_agent( + prompt, json_schema, timeout - (time.time() - start_time) + ) + if result: + return convert_and_return_result(result, type_spec) + logger.error(f"Extraction failed after {time.time() - start_time:.2f} seconds") + return None + + def _try_cached_extraction( + self, prompt: str, json_schema: Optional[JsonSchema] + ) -> Optional[ExtractResponse]: + """ + Attempts to extract data using cached scripts with exponential backoff. + + Args: + prompt: The prompt describing what to extract + json_schema: Optional JSON schema for type validation + + Returns: + ExtractResponse if successful, None otherwise + """ + page = self._get_page() + dto = CachedExtractDTO(url=page.url, prompt=prompt) + scripts = self.logic_engine.get_cached_scripts(dto) + logger.debug(f"Found {len(scripts)} scripts in cache, {scripts}") + if len(scripts) == 0: + logger.debug( + f"No scripts found in cache for prompt: {prompt} in domain: {page.url}" + ) + return None + + def try_cached_extract(): + page = self._get_page() + soup = page._get_soup() + for script in scripts: + res = test_script(script, str(soup), json_schema) + if res is not None: + return ExtractResponse( + status="success", + message="Re-used a preexisting script from cache with the same specifications.", + return_data=res, + created_script=script.script, + ) + return None + + return _attempt_with_backoff_helper( + "cached_extraction", try_cached_extract, CACHE_TIMEOUT + ) + + def _extract_with_agent( + self, prompt: str, json_schema: Optional[JsonSchema], remaining_timeout: float + ) -> Optional[ExtractResponse]: + """ + Attempts to extract data using the extraction agent with exponential backoff. + + Args: + prompt: The prompt describing what to extract + json_schema: Optional JSON schema for type validation + remaining_timeout: Maximum time to spend on extraction + + Returns: + ExtractResponse if successful, None otherwise + """ + + def try_extract_with_agent(): + page = self._get_page() + page_information = page.get_page_information(include_screenshot=True) + extract_dto = ExtractDTO( + page_information=page_information, + prompt=prompt, + return_data_json_schema=json_schema, + use_screenshot=True, + ) + res: ExtractResponse = self.logic_engine.extract(extract_dto) + if res.status == "impossible": + logger.error(f"Impossible to extract data. Reason: {res.message}") + return None + if res.status == "success": + logger.success(f"Extraction successful: '{res.message}'") + return res + return None + + return _attempt_with_backoff_helper( + "extraction_agent", try_extract_with_agent, remaining_timeout + ) + + +def _attempt_with_backoff_helper( + operation_name: str, + operation: Callable, + timeout: float, + backoff_intervals: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0], +) -> Optional[Any]: + """ + Generic helper function that implements exponential backoff for operations. + + Args: + operation_name: Name of the operation for logging + operation: Async function to execute + timeout: Maximum time to spend attempting the operation + backoff_intervals: List of timeouts between attempts + + Returns: + The result of the operation if successful, None otherwise + """ + total_elapsed_time = 0 + start_time = time.time() + for i, current_timeout in enumerate(backoff_intervals): + if total_elapsed_time >= timeout: + logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") + return None + request_start_time = time.time() + result = operation() + request_duration = time.time() - request_start_time + if result: + return result + sleep_duration = max(0, current_timeout - request_duration) + logger.info( + f"{operation_name} attempt {i + 1} failed. Sleeping for {sleep_duration:.2f} seconds" + ) + time.sleep(sleep_duration) + total_elapsed_time = time.time() - start_time + logger.error( + f"All {operation_name} attempts failed after {total_elapsed_time:.2f} seconds" + ) + return None + + +def convert_and_return_result( + res: ExtractResponse, type_spec: Optional[TypeSpec] +) -> TypeSpec: + converted_res = res.return_data + if type_spec is not None: + logger.debug("Converting extraction result to specified type") + converted_res = convert_to_type_spec(type_spec, res.return_data) + logger.info("Extraction process completed successfully") + return converted_res + + +def test_script( + script: Script, raw_html: str, return_data_json_schema: Any +) -> Optional[Any]: + try: + res = execute(script.script, raw_html, return_data_json_schema) + return res + except Exception as e: + logger.debug(f"Script failed with error: {str(e)} ") diff --git a/dendrite/browser/sync_api/mixin/fill_fields.py b/dendrite/browser/sync_api/mixin/fill_fields.py new file mode 100644 index 0000000..4a4880f --- /dev/null +++ b/dendrite/browser/sync_api/mixin/fill_fields.py @@ -0,0 +1,76 @@ +import time +from typing import Any, Dict, Optional +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from dendrite.models.response.interaction_response import InteractionResponse +from ..mixin.get_element import GetElementMixin +from ..protocol.page_protocol import DendritePageProtocol + + +class FillFieldsMixin(GetElementMixin, DendritePageProtocol): + + def fill_fields(self, fields: Dict[str, Any]): + """ + Fills multiple fields on the page with the provided values. + + This method iterates through the given dictionary of fields and their corresponding values, + making a separate fill request for each key-value pair. + + Args: + fields (Dict[str, Any]): A dictionary where each key is a field identifier (e.g., a prompt or selector) + and each value is the content to fill in that field. + + Returns: + None + + Note: + This method will make multiple fill requests, one for each key in the 'fields' dictionary. + """ + for field, value in fields.items(): + prompt = f"I'll be filling in text in several fields with these keys: {fields.keys()} in this page. Get the field best described as '{field}'. I want to fill it with a '{type(value)}' type value." + self.fill(prompt, value) + time.sleep(0.5) + + def fill( + self, + prompt: str, + value: str, + expected_outcome: Optional[str] = None, + use_cache: bool = True, + timeout: int = 15000, + *args, + kwargs={}, + ) -> InteractionResponse: + """ + Fills an element on the page with the provided value based on the given prompt. + + This method combines the functionality of get_element and fill, + allowing for a more concise way to interact with elements on the page. + + Args: + prompt (str): The prompt describing the element to be filled. + value (str): The value to fill the element with. + expected_outcome (Optional[str]): The expected outcome of the fill action. + use_cache (bool, optional): Whether to use cached results for element retrieval. Defaults to True. + max_retries (int, optional): The maximum number of retry attempts for element retrieval. Defaults to 3. + timeout (int, optional): The timeout (in milliseconds) for the fill operation. Defaults to 15000. + *args: Additional positional arguments for the fill operation. + kwargs: Additional keyword arguments for the fill operation. + + Returns: + InteractionResponse: The response from the interaction. + + Raises: + DendriteException: If no suitable element is found or if the fill operation fails. + """ + augmented_prompt = prompt + "\n\nMake sure the element can be filled with text." + element = self.get_element( + augmented_prompt, use_cache=use_cache, timeout=timeout + ) + if not element: + raise DendriteException( + message=f"No element found with the prompt: {prompt}", + screenshot_base64="", + ) + return element.fill( + value, *args, expected_outcome=expected_outcome, timeout=timeout, **kwargs + ) diff --git a/dendrite/browser/sync_api/mixin/get_element.py b/dendrite/browser/sync_api/mixin/get_element.py new file mode 100644 index 0000000..0e32d86 --- /dev/null +++ b/dendrite/browser/sync_api/mixin/get_element.py @@ -0,0 +1,250 @@ +import time +import time +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Literal, + Optional, + Union, + overload, +) +from bs4 import BeautifulSoup +from loguru import logger +from .._utils import _get_all_elements_from_selector_soup +from ..dendrite_element import Element + +if TYPE_CHECKING: + from ..dendrite_page import Page +from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO +from dendrite.models.dto.get_elements_dto import GetElementsDTO +from ..protocol.page_protocol import DendritePageProtocol + +CACHE_TIMEOUT = 5 + + +class GetElementMixin(DendritePageProtocol): + + def get_element( + self, prompt: str, use_cache=True, timeout=15000 + ) -> Optional[Element]: + """ + Retrieves a single Dendrite element based on the provided prompt. + + Args: + prompt (str): The prompt describing the element to be retrieved. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + + Returns: + Element: The retrieved element. + """ + return self._get_element( + prompt, only_one=True, use_cache=use_cache, timeout=timeout / 1000 + ) + + @overload + def _get_element( + self, prompt_or_elements: str, only_one: Literal[True], use_cache: bool, timeout + ) -> Optional[Element]: + """ + Retrieves a single Dendrite element based on the provided prompt. + + Args: + prompt (Union[str, Dict[str, str]]): The prompt describing the element to be retrieved. + only_one (Literal[True]): Indicates that only one element should be retrieved. + use_cache (bool): Whether to use cached results. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + + Returns: + Element: The retrieved element. + """ + + @overload + def _get_element( + self, + prompt_or_elements: str, + only_one: Literal[False], + use_cache: bool, + timeout, + ) -> List[Element]: + """ + Retrieves a list of Dendrite elements based on the provided prompt. + + Args: + prompt (str): The prompt describing the elements to be retrieved. + only_one (Literal[False]): Indicates that multiple elements should be retrieved. + use_cache (bool): Whether to use cached results. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + + Returns: + List[Element]: A list of retrieved elements. + """ + + def _get_element( + self, prompt_or_elements: str, only_one: bool, use_cache: bool, timeout: float + ) -> Union[Optional[Element], List[Element]]: + """ + Retrieves Dendrite elements based on the provided prompt, either a single element or a list of elements. + + This method sends a request with the prompt and retrieves the elements based on the `only_one` flag. + + Args: + prompt_or_elements (Union[str, Dict[str, str]]): The prompt or dictionary of prompts for element retrieval. + only_one (bool): Whether to retrieve only one element or a list of elements. + use_cache (bool): Whether to use cached results. + timeout (int, optional): Maximum time in milliseconds for the entire operation. If use_cache=True, + up to 5000ms will be spent attempting to use cached selectors before falling back to the + find element agent for the remaining time. Defaults to 15000 (15 seconds). + + Returns: + Union[Element, List[Element], ElementsResponse]: The retrieved element, list of elements, or response object. + """ + logger.info(f"Getting element for prompt: '{prompt_or_elements}'") + start_time = time.time() + page = self._get_page() + soup = page._get_soup() + if use_cache: + cached_elements = self._try_cached_selectors( + page, soup, prompt_or_elements, only_one + ) + if cached_elements: + return cached_elements + logger.info( + "Proceeding to use the find element agent to find the requested elements." + ) + res = try_get_element( + self, + prompt_or_elements, + only_one, + remaining_timeout=timeout - (time.time() - start_time), + ) + if res: + return res + logger.error( + f"Failed to retrieve elements within the specified timeout of {timeout} seconds" + ) + return None + + def _try_cached_selectors( + self, page: "Page", soup: BeautifulSoup, prompt: str, only_one: bool + ) -> Union[Optional[Element], List[Element]]: + """ + Attempts to retrieve elements using cached selectors with exponential backoff. + + Args: + page: The current page object + soup: The BeautifulSoup object of the current page + prompt: The prompt to search for + only_one: Whether to return only one element + + Returns: + The found elements if successful, None otherwise + """ + dto = CachedSelectorDTO(url=page.url, prompt=prompt) + selectors = self.logic_engine.get_cached_selectors(dto) + if len(selectors) == 0: + logger.debug("No cached selectors found") + return None + logger.debug("Attempting to use cached selectors with backoff") + str_selectors = list(map(lambda x: x.selector, selectors)) + + def try_cached_selectors(): + return get_elements_from_selectors_soup(page, soup, str_selectors, only_one) + + return _attempt_with_backoff_helper( + "cached_selectors", try_cached_selectors, timeout=CACHE_TIMEOUT + ) + + +def _attempt_with_backoff_helper( + operation_name: str, + operation: Callable, + timeout: float, + backoff_intervals: List[float] = [0.15, 0.45, 1.0, 2.0, 4.0, 8.0], +) -> Optional[Any]: + """ + Generic helper function that implements exponential backoff for operations. + + Args: + operation_name: Name of the operation for logging + operation: Async function to execute + timeout: Maximum time to spend attempting the operation + backoff_intervals: List of timeouts between attempts + + Returns: + The result of the operation if successful, None otherwise + """ + total_elapsed_time = 0 + start_time = time.time() + for i, current_timeout in enumerate(backoff_intervals): + if total_elapsed_time >= timeout: + logger.error(f"Timeout reached after {total_elapsed_time:.2f} seconds") + return None + request_start_time = time.time() + result = operation() + request_duration = time.time() - request_start_time + if result: + return result + sleep_duration = max(0, current_timeout - request_duration) + logger.info( + f"{operation_name} attempt {i + 1} failed. Sleeping for {sleep_duration:.2f} seconds" + ) + time.sleep(sleep_duration) + total_elapsed_time = time.time() - start_time + logger.error( + f"All {operation_name} attempts failed after {total_elapsed_time:.2f} seconds" + ) + return None + + +def try_get_element( + obj: DendritePageProtocol, + prompt_or_elements: Union[str, Dict[str, str]], + only_one: bool, + remaining_timeout: float, +) -> Union[Optional[Element], List[Element]]: + + def _try_get_element(): + page = obj._get_page() + page_information = page.get_page_information() + dto = GetElementsDTO( + page_information=page_information, + prompt=prompt_or_elements, + only_one=only_one, + ) + res = obj.logic_engine.get_element(dto) + if res.status == "impossible": + logger.error( + f"Impossible to get elements for '{prompt_or_elements}'. Reason: {res.message}" + ) + return None + if res.status == "success": + logger.success(f"d[id]: {res.d_id} Selectors:{res.selectors}") + if res.selectors is not None: + return get_elements_from_selectors_soup( + page, page._get_previous_soup(), res.selectors, only_one + ) + return None + + return _attempt_with_backoff_helper( + "find_element_agent", _try_get_element, remaining_timeout + ) + + +def get_elements_from_selectors_soup( + page: "Page", soup: BeautifulSoup, selectors: List[str], only_one: bool +) -> Union[Optional[Element], List[Element]]: + for selector in reversed(selectors): + dendrite_elements = _get_all_elements_from_selector_soup(selector, soup, page) + if len(dendrite_elements) > 0: + return dendrite_elements[0] if only_one else dendrite_elements + return None diff --git a/dendrite/browser/sync_api/mixin/keyboard.py b/dendrite/browser/sync_api/mixin/keyboard.py new file mode 100644 index 0000000..2f1c882 --- /dev/null +++ b/dendrite/browser/sync_api/mixin/keyboard.py @@ -0,0 +1,62 @@ +from typing import Literal, Union +from dendrite.browser._common._exceptions.dendrite_exception import DendriteException +from ..protocol.page_protocol import DendritePageProtocol + + +class KeyboardMixin(DendritePageProtocol): + + def press( + self, + key: Union[ + str, + Literal[ + "Enter", + "Tab", + "Escape", + "Backspace", + "ArrowUp", + "ArrowDown", + "ArrowLeft", + "ArrowRight", + ], + ], + hold_shift: bool = False, + hold_ctrl: bool = False, + hold_alt: bool = False, + hold_cmd: bool = False, + ): + """ + Presses a keyboard key on the active page, optionally with modifier keys. + + Args: + key (Union[str, Literal["Enter", "Tab", "Escape", "Backspace", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight"]]): The main key to be pressed. + hold_shift (bool, optional): Whether to hold the Shift key. Defaults to False. + hold_ctrl (bool, optional): Whether to hold the Control key. Defaults to False. + hold_alt (bool, optional): Whether to hold the Alt key. Defaults to False. + hold_cmd (bool, optional): Whether to hold the Command key (Meta on some systems). Defaults to False. + + Returns: + Any: The result of the key press operation. + + Raises: + DendriteException: If the key press operation fails. + """ + modifiers = [] + if hold_shift: + modifiers.append("Shift") + if hold_ctrl: + modifiers.append("Control") + if hold_alt: + modifiers.append("Alt") + if hold_cmd: + modifiers.append("Meta") + if modifiers: + key = "+".join(modifiers + [key]) + try: + page = self._get_page() + page.keyboard.press(key) + except Exception as e: + raise DendriteException( + message=f"Failed to press key: {key}. Error: {str(e)}", + screenshot_base64="", + ) diff --git a/dendrite/browser/sync_api/mixin/markdown.py b/dendrite/browser/sync_api/mixin/markdown.py new file mode 100644 index 0000000..193a1bb --- /dev/null +++ b/dendrite/browser/sync_api/mixin/markdown.py @@ -0,0 +1,23 @@ +import re +from typing import Optional +from bs4 import BeautifulSoup +from markdownify import markdownify as md +from ..mixin.extract import ExtractionMixin +from ..protocol.page_protocol import DendritePageProtocol + + +class MarkdownMixin(ExtractionMixin, DendritePageProtocol): + + def markdown(self, prompt: Optional[str] = None): + page = self._get_page() + page_information = page.get_page_information() + if prompt: + extract_prompt = f"Create a script that returns the HTML from one element from the DOM that best matches this requested section of the website.\n\nDescription of section: '{prompt}'\n\nWe will be converting your returned HTML to markdown, so just return ONE stringified HTML element and nothing else. It's OK if extra information is present. Example script: 'response_data = soup.find('tag', {{'attribute': 'value'}}).prettify()'" + res = self.extract(extract_prompt) + markdown_text = md(res) + cleaned_markdown = re.sub("\\n{3,}", "\n\n", markdown_text) + return cleaned_markdown + else: + markdown_text = md(page_information.raw_html) + cleaned_markdown = re.sub("\\n{3,}", "\n\n", markdown_text) + return cleaned_markdown diff --git a/dendrite/browser/sync_api/mixin/screenshot.py b/dendrite/browser/sync_api/mixin/screenshot.py new file mode 100644 index 0000000..5cc621e --- /dev/null +++ b/dendrite/browser/sync_api/mixin/screenshot.py @@ -0,0 +1,20 @@ +from ..protocol.page_protocol import DendritePageProtocol + + +class ScreenshotMixin(DendritePageProtocol): + + def screenshot(self, full_page: bool = False) -> str: + """ + Take a screenshot of the current page. + + Args: + full_page (bool, optional): If True, captures the full page. If False, captures only the viewport. Defaults to False. + + Returns: + str: A base64 encoded string of the screenshot in JPEG format. + """ + page = self._get_page() + if full_page: + return page.screenshot_manager.take_full_page_screenshot() + else: + return page.screenshot_manager.take_viewport_screenshot() diff --git a/dendrite/browser/sync_api/mixin/wait_for.py b/dendrite/browser/sync_api/mixin/wait_for.py new file mode 100644 index 0000000..ccc5dfd --- /dev/null +++ b/dendrite/browser/sync_api/mixin/wait_for.py @@ -0,0 +1,53 @@ +import time +import time +from loguru import logger +from dendrite.browser._common._exceptions.dendrite_exception import ( + DendriteException, + PageConditionNotMet, +) +from ..mixin.ask import AskMixin +from ..protocol.page_protocol import DendritePageProtocol + + +class WaitForMixin(AskMixin, DendritePageProtocol): + + def wait_for(self, prompt: str, timeout: float = 30000): + """ + Waits for the condition specified in the prompt to become true by periodically checking the page content. + + This method attempts to retrieve the page information and evaluate whether the specified + condition (provided in the prompt) is met. It continues to retry until the total elapsed time + exceeds the specified timeout. + + Args: + prompt (str): The prompt to determine the condition to wait for on the page. + timeout (float, optional): The maximum time (in milliseconds) to wait for the condition. Defaults to 15000. + + Returns: + Any: The result of the condition evaluation if successful. + + Raises: + PageConditionNotMet: If the condition is not met within the specified timeout. + """ + start_time = time.time() + time.sleep(0.2) + while True: + elapsed_time = (time.time() - start_time) * 1000 + if elapsed_time >= timeout: + break + page = self._get_page() + page_information = page.get_page_information() + prompt_with_instruction = f"Prompt: '{prompt}'\n\nReturn a boolean that determines if the requested information or thing is available on the page. {round(page_information.time_since_frame_navigated, 2)} seconds have passed since the page first loaded." + try: + res = self.ask(prompt_with_instruction, bool) + if res: + return res + except DendriteException as e: + logger.debug(f"Attempt failed: {e.message}") + time.sleep(0.5) + page = self._get_page() + page_information = page.get_page_information() + raise PageConditionNotMet( + message=f"Failed to wait for the requested condition within the {timeout}ms timeout.", + screenshot_base64=page_information.screenshot_base64, + ) diff --git a/dendrite/browser/sync_api/protocol/__init__.py b/dendrite/browser/sync_api/protocol/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/browser/sync_api/protocol/browser_protocol.py b/dendrite/browser/sync_api/protocol/browser_protocol.py new file mode 100644 index 0000000..f708e61 --- /dev/null +++ b/dendrite/browser/sync_api/protocol/browser_protocol.py @@ -0,0 +1,61 @@ +from typing import TYPE_CHECKING, Optional, Protocol, Union +from typing_extensions import Literal +from dendrite.browser.remote import Providers + +if TYPE_CHECKING: + from ..dendrite_browser import Dendrite +from playwright.sync_api import Browser, Download, Playwright +from ..types import PlaywrightPage + + +class BrowserProtocol(Protocol): + + def __init__(self, settings: Providers) -> None: ... + + def get_download( + self, dendrite_browser: "Dendrite", pw_page: PlaywrightPage, timeout: float + ) -> Download: + """ + Retrieves the download event from the browser. + + Returns: + Download: The download event. + + Raises: + Exception: If there is an issue retrieving the download event. + """ + ... + + def start_browser(self, playwright: Playwright, pw_options: dict) -> Browser: + """ + Starts the browser session. + + Args: + playwright: The playwright instance + pw_options: Playwright launch options + + Returns: + Browser: A Browser instance + """ + ... + + def configure_context(self, browser: "Dendrite") -> None: + """ + Configures the browser context. + + Args: + browser (Dendrite): The browser to configure. + + Raises: + Exception: If there is an issue configuring the browser context. + """ + ... + + def stop_session(self) -> None: + """ + Stops the browser session. + + Raises: + Exception: If there is an issue stopping the browser session. + """ + ... diff --git a/dendrite/browser/sync_api/protocol/download_protocol.py b/dendrite/browser/sync_api/protocol/download_protocol.py new file mode 100644 index 0000000..8a68843 --- /dev/null +++ b/dendrite/browser/sync_api/protocol/download_protocol.py @@ -0,0 +1,20 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, Union +from playwright.sync_api import Download + + +class DownloadInterface(ABC, Download): + + def __init__(self, download: Download): + self._download = download + + def __getattribute__(self, name: str) -> Any: + try: + return super().__getattribute__(name) + except AttributeError: + return getattr(self._download, name) + + @abstractmethod + def save_as(self, path: Union[str, Path]) -> None: + pass diff --git a/dendrite/browser/sync_api/protocol/page_protocol.py b/dendrite/browser/sync_api/protocol/page_protocol.py new file mode 100644 index 0000000..d12b839 --- /dev/null +++ b/dendrite/browser/sync_api/protocol/page_protocol.py @@ -0,0 +1,21 @@ +from typing import TYPE_CHECKING, Protocol +from dendrite.logic import LogicEngine + +if TYPE_CHECKING: + from ..dendrite_browser import Dendrite + from ..dendrite_page import Page + + +class DendritePageProtocol(Protocol): + """ + Protocol that specifies the required methods and attributes + for the `ExtractionMixin` to work. + """ + + @property + def logic_engine(self) -> LogicEngine: ... + + @property + def dendrite_browser(self) -> "Dendrite": ... + + def _get_page(self) -> "Page": ... diff --git a/dendrite/browser/sync_api/types.py b/dendrite/browser/sync_api/types.py new file mode 100644 index 0000000..de26bef --- /dev/null +++ b/dendrite/browser/sync_api/types.py @@ -0,0 +1,12 @@ +import inspect +from typing import Any, Dict, Literal, Type, TypeVar, Union +from playwright.sync_api import Page +from pydantic import BaseModel + +Interaction = Literal["click", "fill", "hover"] +T = TypeVar("T") +PydanticModel = TypeVar("PydanticModel", bound=BaseModel) +PrimitiveTypes = PrimitiveTypes = Union[Type[bool], Type[int], Type[float], Type[str]] +JsonSchema = Dict[str, Any] +TypeSpec = Union[PrimitiveTypes, PydanticModel, JsonSchema] +PlaywrightPage = Page diff --git a/dendrite/logic/__init__.py b/dendrite/logic/__init__.py new file mode 100644 index 0000000..4c2737c --- /dev/null +++ b/dendrite/logic/__init__.py @@ -0,0 +1,4 @@ +from .async_logic_engine import AsyncLogicEngine +from .sync_logic_engine import LogicEngine + +__all__ = ["LogicEngine", "AsyncLogicEngine"] diff --git a/dendrite/logic/ask/__init__.py b/dendrite/logic/ask/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/ask/ask.py b/dendrite/logic/ask/ask.py index 25ea9c4..af7e71e 100644 --- a/dendrite/logic/ask/ask.py +++ b/dendrite/logic/ask/ask.py @@ -3,12 +3,10 @@ import json_repair from jsonschema import validate - from openai.types.chat.chat_completion_content_part_param import ( ChatCompletionContentPartParam, ) - from dendrite.logic.config import Config from dendrite.logic.llm.agent import Agent, Message from dendrite.models.dto.ask_page_dto import AskPageDTO diff --git a/dendrite/logic/interfaces/async_api.py b/dendrite/logic/async_logic_engine.py similarity index 81% rename from dendrite/logic/interfaces/async_api.py rename to dendrite/logic/async_logic_engine.py index 5916ab8..38915bd 100644 --- a/dendrite/logic/interfaces/async_api.py +++ b/dendrite/logic/async_logic_engine.py @@ -1,39 +1,25 @@ from typing import List, Optional, Protocol +from dendrite.logic.ask import ask from dendrite.logic.config import Config +from dendrite.logic.extract import extract from dendrite.logic.get_element import get_element +from dendrite.logic.verify_interaction import verify_interaction from dendrite.models.dto.ask_page_dto import AskPageDTO from dendrite.models.dto.cached_extract_dto import CachedExtractDTO +from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO from dendrite.models.dto.extract_dto import ExtractDTO from dendrite.models.dto.get_elements_dto import GetElementsDTO -from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO from dendrite.models.dto.make_interaction_dto import VerifyActionDTO from dendrite.models.response.ask_page_response import AskPageResponse - from dendrite.models.response.extract_response import ExtractResponse from dendrite.models.response.get_element_response import GetElementResponse from dendrite.models.response.interaction_response import InteractionResponse - -from dendrite.logic.ask import ask -from dendrite.logic.extract import extract -from dendrite.logic import verify_interaction from dendrite.models.scripts import Script from dendrite.models.selector import Selector -class LogicAPIProtocol(Protocol): - - async def get_element(self, dto: GetElementsDTO) -> GetElementResponse: ... - - async def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: ... - - async def extract(self, dto: ExtractDTO) -> ExtractResponse: ... - - async def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... - - -class AsyncProtocol(LogicAPIProtocol): - +class AsyncLogicEngine: def __init__(self, config: Config): self._config = config diff --git a/dendrite/logic/cache/__init__.py b/dendrite/logic/cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/cache/element_cache.py b/dendrite/logic/cache/element_cache.py index 05e8106..09a2876 100644 --- a/dendrite/logic/cache/element_cache.py +++ b/dendrite/logic/cache/element_cache.py @@ -1,4 +1,5 @@ -from typing import Generic, TypeVar, TypedDict +from typing import Generic, TypedDict, TypeVar + from pydantic import BaseModel from dendrite.logic.cache.file_cache import FileCache diff --git a/dendrite/logic/cache/file_cache.py b/dendrite/logic/cache/file_cache.py index d5e165a..2e83079 100644 --- a/dendrite/logic/cache/file_cache.py +++ b/dendrite/logic/cache/file_cache.py @@ -2,7 +2,7 @@ import threading from hashlib import md5 from pathlib import Path -from typing import Dict, Generic, Type, TypeVar, Union, Any, Mapping +from typing import Any, Dict, Generic, Mapping, Type, TypeVar, Union from pydantic import BaseModel diff --git a/dendrite/logic/code/__init__.py b/dendrite/logic/code/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/config.py b/dendrite/logic/config.py index 6295b93..b69daf7 100644 --- a/dendrite/logic/config.py +++ b/dendrite/logic/config.py @@ -1,12 +1,13 @@ from pathlib import Path from typing import Optional, Union + +from playwright.async_api import StorageState + from dendrite.logic.cache.file_cache import FileCache from dendrite.logic.llm.config import LLMConfig from dendrite.models.scripts import Script from dendrite.models.selector import Selector -from playwright.async_api import StorageState - class Config: def __init__( diff --git a/dendrite/logic/dom/__init__.py b/dendrite/logic/dom/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/extract/__init__.py b/dendrite/logic/extract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/extract/extract_agent.py b/dendrite/logic/extract/extract_agent.py index 3cfce36..d5baf16 100644 --- a/dendrite/logic/extract/extract_agent.py +++ b/dendrite/logic/extract/extract_agent.py @@ -3,8 +3,9 @@ import sys from typing import List, Union -from dendrite import logger +from bs4 import BeautifulSoup +from dendrite import logger from dendrite.logic.cache.utils import save_script from dendrite.logic.config import Config from dendrite.logic.dom.strip import mild_strip @@ -12,20 +13,16 @@ LARGE_HTML_CHAR_TRUNCATE_LEN, create_script_prompt_segmented_html, ) - from dendrite.logic.extract.scroll_agent import ScrollAgent -from dendrite.logic.llm.agent import Agent from dendrite.logic.get_element.hanifi_search import get_expanded_dom +from dendrite.logic.llm.agent import Agent, Message from dendrite.logic.llm.token_count import token_count from dendrite.models.dto.extract_dto import ExtractDTO from dendrite.models.page_information import PageInformation -from dendrite.logic.llm.agent import Message - -from bs4 import BeautifulSoup - from dendrite.models.response.extract_response import ExtractResponse -from ..code.code_session import CodeSession + from ..ask.image import segment_image +from ..code.code_session import CodeSession class ExtractAgent(Agent): diff --git a/dendrite/logic/extract/scroll_agent.py b/dendrite/logic/extract/scroll_agent.py index 7c72225..9038557 100644 --- a/dendrite/logic/extract/scroll_agent.py +++ b/dendrite/logic/extract/scroll_agent.py @@ -3,12 +3,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import List, Literal, Optional + +from loguru import logger from openai.types.chat.chat_completion_content_part_param import ( ChatCompletionContentPartParam, ) -from loguru import logger - from dendrite.logic.llm.agent import Agent, Message from dendrite.logic.llm.config import LLMConfig from dendrite.models.page_information import PageInformation diff --git a/dendrite/logic/factory.py b/dendrite/logic/factory.py deleted file mode 100644 index d4fd85f..0000000 --- a/dendrite/logic/factory.py +++ /dev/null @@ -1,15 +0,0 @@ -# from typing import Literal, Optional - -# from dendrite.logic.interfaces import AsyncProtocol - - -# class BrowserAPIFactory: -# @staticmethod -# def create_browser_api( -# mode: Literal["local", "remote"], -# session_id: Optional[str] = None -# ) -> LogicAPIProtocol': -# if mode == "local": -# return LocalBrowserAPI() -# else: -# return BrowserAPIClient(api_config, session_id) diff --git a/dendrite/logic/get_element/__init__.py b/dendrite/logic/get_element/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/get_element/agents/segment_agent.py b/dendrite/logic/get_element/agents/segment_agent.py index 61c1d85..574c821 100644 --- a/dendrite/logic/get_element/agents/segment_agent.py +++ b/dendrite/logic/get_element/agents/segment_agent.py @@ -5,8 +5,10 @@ from annotated_types import Len from loguru import logger from pydantic import BaseModel, ValidationError + from dendrite.logic.llm.agent import Agent from dendrite.logic.llm.config import LLMConfig + from .prompts import SEGMENT_PROMPT diff --git a/dendrite/logic/get_element/get_element.py b/dendrite/logic/get_element/get_element.py index 2df4497..58b65af 100644 --- a/dendrite/logic/get_element/get_element.py +++ b/dendrite/logic/get_element/get_element.py @@ -2,6 +2,7 @@ from bs4 import BeautifulSoup, Tag from loguru import logger + from dendrite.logic.config import Config from dendrite.logic.dom.css import check_if_selector_successful, find_css_selector from dendrite.logic.dom.strip import remove_hidden_elements diff --git a/dendrite/logic/interfaces/__init__.py b/dendrite/logic/interfaces/__init__.py deleted file mode 100644 index 3f4d492..0000000 --- a/dendrite/logic/interfaces/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .async_api import AsyncProtocol -from .sync_api import SyncProtocol - - -__all__ = [ - "AsyncProtocol", - "SyncProtocol", -] diff --git a/dendrite/logic/interfaces/cache.py b/dendrite/logic/interfaces/cache.py deleted file mode 100644 index aa368d6..0000000 --- a/dendrite/logic/interfaces/cache.py +++ /dev/null @@ -1,5 +0,0 @@ -from typing import Generic, Protocol, TypeVar, Union, overload - -from pydantic import BaseModel - -T = TypeVar("T", bound=BaseModel) diff --git a/dendrite/logic/llm/__init__.py b/dendrite/logic/llm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/interfaces/sync_api.py b/dendrite/logic/sync_logic_engine.py similarity index 85% rename from dendrite/logic/interfaces/sync_api.py rename to dendrite/logic/sync_logic_engine.py index 0537b6f..ccd2dd5 100644 --- a/dendrite/logic/interfaces/sync_api.py +++ b/dendrite/logic/sync_logic_engine.py @@ -1,10 +1,13 @@ import asyncio -from concurrent.futures import ThreadPoolExecutor import threading -from typing import Any, Coroutine, List, Protocol, TypeVar +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Coroutine, List, TypeVar +from dendrite.logic.ask import ask from dendrite.logic.config import Config +from dendrite.logic.extract import extract from dendrite.logic.get_element import get_element +from dendrite.logic.verify_interaction import verify_interaction from dendrite.models.dto.ask_page_dto import AskPageDTO from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.dto.cached_selector_dto import CachedSelectorDTO @@ -12,18 +15,12 @@ from dendrite.models.dto.get_elements_dto import GetElementsDTO from dendrite.models.dto.make_interaction_dto import VerifyActionDTO from dendrite.models.response.ask_page_response import AskPageResponse - from dendrite.models.response.extract_response import ExtractResponse from dendrite.models.response.get_element_response import GetElementResponse from dendrite.models.response.interaction_response import InteractionResponse - -from dendrite.logic.ask import ask -from dendrite.logic.extract import extract -from dendrite.logic import verify_interaction from dendrite.models.scripts import Script from dendrite.models.selector import Selector - T = TypeVar("T") @@ -52,18 +49,7 @@ def run_in_new_loop(): return asyncio.run_coroutine_threadsafe(coroutine, loop).result() -class LogicAPIProtocol(Protocol): - - def get_element(self, dto: GetElementsDTO) -> GetElementResponse: ... - - def verify_action(self, dto: VerifyActionDTO) -> InteractionResponse: ... - - def extract(self, dto: ExtractDTO) -> ExtractResponse: ... - - def ask_page(self, dto: AskPageDTO) -> AskPageResponse: ... - - -class SyncProtocol(LogicAPIProtocol): +class LogicEngine: def __init__(self, config: Config): self._config = config diff --git a/dendrite/logic/verify_interaction/__init__.py b/dendrite/logic/verify_interaction/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/logic/verify_interaction.py b/dendrite/logic/verify_interaction/verify_interaction.py similarity index 100% rename from dendrite/logic/verify_interaction.py rename to dendrite/logic/verify_interaction/verify_interaction.py diff --git a/dendrite/models/__init__.py b/dendrite/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/models/dto/__init__.py b/dendrite/models/dto/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/models/response/__init__.py b/dendrite/models/response/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dendrite/models/response/ask_page_response.py b/dendrite/models/response/ask_page_response.py index 4ec747a..86c6865 100644 --- a/dendrite/models/response/ask_page_response.py +++ b/dendrite/models/response/ask_page_response.py @@ -1,6 +1,6 @@ from typing import Generic, Literal, TypeVar -from pydantic import BaseModel +from pydantic import BaseModel T = TypeVar("T") diff --git a/dendrite/models/status.py b/dendrite/models/status.py index 0068d7d..427449d 100644 --- a/dendrite/models/status.py +++ b/dendrite/models/status.py @@ -1,4 +1,3 @@ from typing import Literal - Status = Literal["success", "failed", "loading", "impossible"] diff --git a/dendrite/remote/__init__.py b/dendrite/remote/__init__.py new file mode 100644 index 0000000..22f7310 --- /dev/null +++ b/dendrite/remote/__init__.py @@ -0,0 +1,6 @@ +from dendrite.browser.remote import BrowserbaseConfig, BrowserlessConfig, Providers + +__all__ = [ + "BrowserbaseConfig", + "BrowserlessConfig", +] \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index e5d6827..d1c159b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2285,6 +2285,20 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-file" +version = "2.1.0" +description = "File transport adapter for Requests" +optional = false +python-versions = "*" +files = [ + {file = "requests_file-2.1.0-py2.py3-none-any.whl", hash = "sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c"}, + {file = "requests_file-2.1.0.tar.gz", hash = "sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658"}, +] + +[package.dependencies] +requests = ">=1.0.0" + [[package]] name = "rpds-py" version = "0.21.0" @@ -2464,6 +2478,27 @@ requests = ">=2.26.0" [package.extras] blobfile = ["blobfile (>=2)"] +[[package]] +name = "tldextract" +version = "5.1.3" +description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well." +optional = false +python-versions = ">=3.9" +files = [ + {file = "tldextract-5.1.3-py3-none-any.whl", hash = "sha256:78de310cc2ca018692de5ddf320f9d6bd7c5cf857d0fd4f2175f0cdf4440ea75"}, + {file = "tldextract-5.1.3.tar.gz", hash = "sha256:d43c7284c23f5dc8a42fd0fee2abede2ff74cc622674e4cb07f514ab3330c338"}, +] + +[package.dependencies] +filelock = ">=3.0.8" +idna = "*" +requests = ">=2.1.0" +requests-file = ">=1.4" + +[package.extras] +release = ["build", "twine"] +testing = ["mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ruff", "syrupy", "tox", "tox-uv", "types-filelock", "types-requests"] + [[package]] name = "tokenizers" version = "0.20.3" @@ -2796,4 +2831,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f08f549a61b2efdf0e4a859882fe6009ecc47d4cddff3fbdffec73e7ed5ad28a" +content-hash = "70dde474fd4580a2207cbeb0fdbe3c763891e0cd19317908e9b346fa5cb9f6b1" diff --git a/scripts/generate_sync.py b/scripts/generate_sync.py index 7d607a9..7d27ef0 100644 --- a/scripts/generate_sync.py +++ b/scripts/generate_sync.py @@ -289,7 +289,7 @@ def get_uncommitted_diff(folder): "AsyncPage": "Page", "AsyncDendriteRemoteBrowser": "DendriteRemoteBrowser", "AsyncElementsResponse": "ElementsResponse", - "AsyncProtocol": "SyncProtocol", + "AsyncLogicEngine": "LogicEngine", } if check_for_uncommitted_changes(target_dir): diff --git a/tests/tests_async/conftest.py b/tests/tests_async/conftest.py index 1263fb6..49499da 100644 --- a/tests/tests_async/conftest.py +++ b/tests/tests_async/conftest.py @@ -1,13 +1,8 @@ import pytest -import asyncio - import pytest_asyncio -from dendrite.browser.async_api._core.dendrite_browser import ( - AsyncDendrite, -) -from dendrite.browser.remote import ( - BrowserbaseConfig, -) # Import your class here + +from dendrite import AsyncDendrite +from dendrite.remote import BrowserbaseConfig # Import your class here @pytest_asyncio.fixture(scope="session") @@ -18,9 +13,6 @@ async def dendrite_browser(): The fixture has a session scope, so it will only be initialized once for the entire test session. """ async with AsyncDendrite( - openai_api_key="your_openai_api_key", - dendrite_api_key="your_dendrite_api_key", - anthropic_api_key="your_anthropic_api_key", playwright_options={"headless": True}, ) as browser: yield browser # Provide the browser to tests @@ -34,9 +26,6 @@ async def browserbase(): The fixture has a session scope, so it will only be initialized once for the entire test session. """ async with AsyncDendrite( - openai_api_key="your_openai_api_key", - dendrite_api_key="your_dendrite_api_key", - anthropic_api_key="your_anthropic_api_key", playwright_options={"headless": True}, remote_config=BrowserbaseConfig(), ) as browser: diff --git a/tests/tests_async/test_download.py b/tests/tests_async/test_download.py index e4575fa..a1b39af 100644 --- a/tests/tests_async/test_download.py +++ b/tests/tests_async/test_download.py @@ -1,8 +1,8 @@ -import asyncio import os + import pytest -from dendrite.browser.async_api._core.dendrite_browser import AsyncDendrite +from dendrite import AsyncDendrite pytest_plugins = ("pytest_asyncio",) diff --git a/tests/tests_async/tests.py b/tests/tests_async/tests.py index 7e7915a..841e803 100644 --- a/tests/tests_async/tests.py +++ b/tests/tests_async/tests.py @@ -1,4 +1,5 @@ import pytest + from dendrite import AsyncDendrite diff --git a/tests/tests_sync/conftest.py b/tests/tests_sync/conftest.py index 8fa6ae7..fd9fc87 100644 --- a/tests/tests_sync/conftest.py +++ b/tests/tests_sync/conftest.py @@ -1,5 +1,6 @@ import pytest -from dendrite.browser.sync_api import Dendrite + +from dendrite import Dendrite @pytest.fixture(scope="session") @@ -10,9 +11,6 @@ def dendrite_browser(): The fixture has a session scope, so it will only be initialized once for the entire test session. """ browser = Dendrite( - openai_api_key="your_openai_api_key", - dendrite_api_key="your_dendrite_api_key", - anthropic_api_key="your_anthropic_api_key", playwright_options={"headless": True}, ) # Launch the browser diff --git a/tests/tests_sync/test_context.py b/tests/tests_sync/test_context.py index 10a7dd6..06713dd 100644 --- a/tests/tests_sync/test_context.py +++ b/tests/tests_sync/test_context.py @@ -1,13 +1,9 @@ # content of test_tmp_path.py -import os -from dendrite.browser.sync_api import Dendrite +from dendrite import Dendrite def test_context_manager(): with Dendrite( - openai_api_key="your_openai_api_key", - dendrite_api_key="your_dendrite_api_key", - anthropic_api_key="your_anthropic_api_key", playwright_options={"headless": True}, ) as browser: browser.goto("https://dendrite.systems") diff --git a/tests/tests_sync/test_download.py b/tests/tests_sync/test_download.py index d3279bb..db02e2e 100644 --- a/tests/tests_sync/test_download.py +++ b/tests/tests_sync/test_download.py @@ -1,6 +1,7 @@ # content of test_tmp_path.py import os -from dendrite.browser.sync_api import Dendrite + +from dendrite import Dendrite def test_download(dendrite_browser: Dendrite, tmp_path): From 5ed25abb956c7829d0ada92be8580109d3508629 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Thu, 5 Dec 2024 16:40:29 +0100 Subject: [PATCH 13/18] update to test multiple cached selectors and scripts --- dendrite/browser/async_api/_utils.py | 1 - .../browser/async_api/dendrite_browser.py | 2 +- dendrite/browser/async_api/mixin/extract.py | 5 +- .../browser/async_api/mixin/get_element.py | 4 +- dendrite/browser/sync_api/_utils.py | 1 - .../sync_api/browser_impl/local/__init__.py | 0 dendrite/browser/sync_api/dendrite_browser.py | 2 +- dendrite/browser/sync_api/mixin/extract.py | 4 +- .../browser/sync_api/mixin/get_element.py | 3 +- dendrite/logic/cache/element_cache.py | 9 -- dendrite/logic/cache/extract_cache.py | 5 - dendrite/logic/cache/file_cache.py | 115 ++++++++++++++---- dendrite/logic/cache/storage_cache.py | 0 dendrite/logic/cache/utils.py | 14 ++- .../extract/{cached_script.py => cache.py} | 32 +++-- dendrite/logic/extract/extract.py | 17 +-- dendrite/logic/extract/extract_agent.py | 4 +- .../{cached_selector.py => cache.py} | 6 +- dendrite/logic/get_element/get_element.py | 4 +- dendrite/remote/__init__.py | 2 +- 20 files changed, 154 insertions(+), 76 deletions(-) delete mode 100644 dendrite/browser/sync_api/browser_impl/local/__init__.py delete mode 100644 dendrite/logic/cache/element_cache.py delete mode 100644 dendrite/logic/cache/extract_cache.py delete mode 100644 dendrite/logic/cache/storage_cache.py rename dendrite/logic/extract/{cached_script.py => cache.py} (53%) rename dendrite/logic/get_element/{cached_selector.py => cache.py} (84%) diff --git a/dendrite/browser/async_api/_utils.py b/dendrite/browser/async_api/_utils.py index 6bb14c8..3ccf4f4 100644 --- a/dendrite/browser/async_api/_utils.py +++ b/dendrite/browser/async_api/_utils.py @@ -70,7 +70,6 @@ async def get_iframe_path(frame: Frame): mild_strip_in_place(frame_tree) merge_iframe_to_page(iframe_id, page_soup, frame_tree) except Error as e: - logger.debug(f"Error processing frame {iframe_id}: {e}") continue diff --git a/dendrite/browser/async_api/dendrite_browser.py b/dendrite/browser/async_api/dendrite_browser.py index 8d6a4d3..ca51778 100644 --- a/dendrite/browser/async_api/dendrite_browser.py +++ b/dendrite/browser/async_api/dendrite_browser.py @@ -523,7 +523,7 @@ async def setup_auth( async def _get_domain_storage_state(self, domain: str) -> Optional[StorageState]: """Get storage state for a specific domain""" - return self._config.storage_cache.get({"domain": domain}) + return self._config.storage_cache.get({"domain": domain}, index=0) async def _merge_storage_states(self, states: List[StorageState]) -> StorageState: """Merge multiple storage states into one""" diff --git a/dendrite/browser/async_api/mixin/extract.py b/dendrite/browser/async_api/mixin/extract.py index f3ec334..b718567 100644 --- a/dendrite/browser/async_api/mixin/extract.py +++ b/dendrite/browser/async_api/mixin/extract.py @@ -157,6 +157,7 @@ async def _try_cached_extraction( ) -> Optional[ExtractResponse]: """ Attempts to extract data using cached scripts with exponential backoff. + Only tries up to 5 most recent scripts. Args: prompt: The prompt describing what to extract @@ -178,7 +179,9 @@ async def _try_cached_extraction( async def try_cached_extract(): page = await self._get_page() soup = await page._get_soup() - for script in scripts: + # Take at most the last 5 scripts + recent_scripts = scripts[-min(5, len(scripts)) :] + for script in recent_scripts: res = await test_script(script, str(soup), json_schema) if res is not None: return ExtractResponse( diff --git a/dendrite/browser/async_api/mixin/get_element.py b/dendrite/browser/async_api/mixin/get_element.py index 44878a9..51f8235 100644 --- a/dendrite/browser/async_api/mixin/get_element.py +++ b/dendrite/browser/async_api/mixin/get_element.py @@ -186,7 +186,9 @@ async def _try_cached_selectors( return None logger.debug("Attempting to use cached selectors with backoff") - str_selectors = list(map(lambda x: x.selector, selectors)) + # Take at most the last 5 selectors + recent_selectors = selectors[-min(5, len(selectors)) :] + str_selectors = list(map(lambda x: x.selector, recent_selectors)) async def try_cached_selectors(): return await get_elements_from_selectors_soup( diff --git a/dendrite/browser/sync_api/_utils.py b/dendrite/browser/sync_api/_utils.py index eecf191..24f2b8a 100644 --- a/dendrite/browser/sync_api/_utils.py +++ b/dendrite/browser/sync_api/_utils.py @@ -58,7 +58,6 @@ def get_iframe_path(frame: Frame): mild_strip_in_place(frame_tree) merge_iframe_to_page(iframe_id, page_soup, frame_tree) except Error as e: - logger.debug(f"Error processing frame {iframe_id}: {e}") continue diff --git a/dendrite/browser/sync_api/browser_impl/local/__init__.py b/dendrite/browser/sync_api/browser_impl/local/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/browser/sync_api/dendrite_browser.py b/dendrite/browser/sync_api/dendrite_browser.py index b5fdd64..6747a19 100644 --- a/dendrite/browser/sync_api/dendrite_browser.py +++ b/dendrite/browser/sync_api/dendrite_browser.py @@ -462,7 +462,7 @@ def setup_auth( def _get_domain_storage_state(self, domain: str) -> Optional[StorageState]: """Get storage state for a specific domain""" - return self._config.storage_cache.get({"domain": domain}) + return self._config.storage_cache.get({"domain": domain}, index=0) def _merge_storage_states(self, states: List[StorageState]) -> StorageState: """Merge multiple storage states into one""" diff --git a/dendrite/browser/sync_api/mixin/extract.py b/dendrite/browser/sync_api/mixin/extract.py index 292fb4f..e5bf411 100644 --- a/dendrite/browser/sync_api/mixin/extract.py +++ b/dendrite/browser/sync_api/mixin/extract.py @@ -141,6 +141,7 @@ def _try_cached_extraction( ) -> Optional[ExtractResponse]: """ Attempts to extract data using cached scripts with exponential backoff. + Only tries up to 5 most recent scripts. Args: prompt: The prompt describing what to extract @@ -162,7 +163,8 @@ def _try_cached_extraction( def try_cached_extract(): page = self._get_page() soup = page._get_soup() - for script in scripts: + recent_scripts = scripts[-min(5, len(scripts)) :] + for script in recent_scripts: res = test_script(script, str(soup), json_schema) if res is not None: return ExtractResponse( diff --git a/dendrite/browser/sync_api/mixin/get_element.py b/dendrite/browser/sync_api/mixin/get_element.py index 0e32d86..84e2b37 100644 --- a/dendrite/browser/sync_api/mixin/get_element.py +++ b/dendrite/browser/sync_api/mixin/get_element.py @@ -155,7 +155,8 @@ def _try_cached_selectors( logger.debug("No cached selectors found") return None logger.debug("Attempting to use cached selectors with backoff") - str_selectors = list(map(lambda x: x.selector, selectors)) + recent_selectors = selectors[-min(5, len(selectors)) :] + str_selectors = list(map(lambda x: x.selector, recent_selectors)) def try_cached_selectors(): return get_elements_from_selectors_soup(page, soup, str_selectors, only_one) diff --git a/dendrite/logic/cache/element_cache.py b/dendrite/logic/cache/element_cache.py deleted file mode 100644 index 09a2876..0000000 --- a/dendrite/logic/cache/element_cache.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Generic, TypedDict, TypeVar - -from pydantic import BaseModel - -from dendrite.logic.cache.file_cache import FileCache -from dendrite.models.scripts import Script -from dendrite.models.selector import Selector - -element_cache = FileCache(Selector, "./cache/get_element.json") diff --git a/dendrite/logic/cache/extract_cache.py b/dendrite/logic/cache/extract_cache.py deleted file mode 100644 index 2d5d8f3..0000000 --- a/dendrite/logic/cache/extract_cache.py +++ /dev/null @@ -1,5 +0,0 @@ -from dendrite.models.scripts import Script - -from .file_cache import FileCache - -ExtractCache = FileCache(Script, "./cache/extract.json") diff --git a/dendrite/logic/cache/file_cache.py b/dendrite/logic/cache/file_cache.py index 2e83079..b56bc18 100644 --- a/dendrite/logic/cache/file_cache.py +++ b/dendrite/logic/cache/file_cache.py @@ -2,7 +2,18 @@ import threading from hashlib import md5 from pathlib import Path -from typing import Any, Dict, Generic, Mapping, Type, TypeVar, Union +from typing import ( + Any, + Dict, + Generic, + List, + Mapping, + Type, + TypeVar, + Union, + Optional, + overload, +) from pydantic import BaseModel @@ -16,7 +27,7 @@ def __init__( self.filepath = Path(filepath) self.model_class = model_class self.lock = threading.RLock() - self.cache: Dict[str, T] = {} + self.cache: Dict[str, List[T]] = {} # Create file if it doesn't exist if not self.filepath.exists(): @@ -34,47 +45,103 @@ def _load_cache(self) -> None: # Convert each entry based on model_class type self.cache = {} - for k, v in raw_dict.items(): - if issubclass(self.model_class, BaseModel): - self.cache[k] = self.model_class.model_validate_json( - json.dumps(v) - ) - else: - # For any Mapping type (dict, TypedDict, etc) - self.cache[k] = v + for k, v_list in raw_dict.items(): + if not isinstance(v_list, list): + v_list = [v_list] # Convert old single-value format to list + + self.cache[k] = [] + for v in v_list: + if issubclass(self.model_class, BaseModel): + self.cache[k].append( + self.model_class.model_validate_json(json.dumps(v)) + ) + else: + # For any Mapping type (dict, TypedDict, etc) + self.cache[k].append(v) except (json.JSONDecodeError, FileNotFoundError): self.cache = {} - def _save_cache(self, cache_dict: Dict[str, T]) -> None: + def _save_cache(self, cache_dict: Dict[str, List[T]]) -> None: """Save cache to file""" with self.lock: # Convert entries based on their type serializable_dict = {} - for k, v in cache_dict.items(): - if isinstance(v, BaseModel): - serializable_dict[k] = json.loads(v.model_dump_json()) - elif isinstance(v, Mapping): - serializable_dict[k] = dict(v) # Convert any Mapping to dict - else: - raise ValueError(f"Unsupported type for cache value: {type(v)}") + for k, v_list in cache_dict.items(): + serializable_dict[k] = [] + for v in v_list: + if isinstance(v, BaseModel): + serializable_dict[k].append(json.loads(v.model_dump_json())) + elif isinstance(v, Mapping): + serializable_dict[k].append( + dict(v) + ) # Convert any Mapping to dict + else: + raise ValueError(f"Unsupported type for cache value: {type(v)}") self.filepath.write_text(json.dumps(serializable_dict, indent=2)) - def get(self, key: Union[str, Dict[str, str]]) -> Union[T, None]: + @overload + def get( + self, key: Union[str, Dict[str, str]], index: None = None + ) -> Optional[List[T]]: ... + + @overload + def get(self, key: Union[str, Dict[str, str]], index: int) -> Optional[T]: ... + + def get( + self, key: Union[str, Dict[str, str]], index: Optional[int] = None + ) -> Union[T, List[T], None]: + """ + Get cached values for a key. If index is provided, returns that specific item. + If index is None, returns the full list of items. + Returns None if key doesn't exist or index is out of range. + """ + hashed_key = self.hash(key) + values = self.cache.get(hashed_key, []) + + if index is not None: + return values[index] if 0 <= index < len(values) else None + return values if values else None + + def set(self, key: Union[str, Dict[str, str]], values: Union[T, List[T]]) -> None: + """ + Replace all values for a key with new value(s). + If a single value is provided, it will be wrapped in a list. + """ hashed_key = self.hash(key) - return self.cache.get(hashed_key) + with self.lock: + if isinstance(values, list): + self.cache[hashed_key] = values + else: + self.cache[hashed_key] = [values] + self._save_cache(self.cache) - def set(self, key: Union[str, Dict[str, str]], value: T) -> None: + def append(self, key: Union[str, Dict[str, str]], value: T) -> None: + """ + Append a single value to the list of values for a key. + Creates a new list if the key doesn't exist. + """ hashed_key = self.hash(key) with self.lock: - self.cache[hashed_key] = value + if hashed_key not in self.cache: + self.cache[hashed_key] = [] + self.cache[hashed_key].append(value) self._save_cache(self.cache) - def delete(self, key: str) -> None: + def delete(self, key: str, index: Optional[int] = None) -> None: + """ + Delete cached value(s). If index is provided, only that item is deleted. + If index is None, all items for the key are deleted. + """ hashed_key = self.hash(key) with self.lock: if hashed_key in self.cache: - del self.cache[hashed_key] + if index is not None and 0 <= index < len(self.cache[hashed_key]): + del self.cache[hashed_key][index] + if not self.cache[hashed_key]: # Remove key if list is empty + del self.cache[hashed_key] + else: + del self.cache[hashed_key] self._save_cache(self.cache) def hash(self, key: Union[str, Dict]) -> str: diff --git a/dendrite/logic/cache/storage_cache.py b/dendrite/logic/cache/storage_cache.py deleted file mode 100644 index e69de29..0000000 diff --git a/dendrite/logic/cache/utils.py b/dendrite/logic/cache/utils.py index e5796fb..6e9a934 100644 --- a/dendrite/logic/cache/utils.py +++ b/dendrite/logic/cache/utils.py @@ -1,19 +1,21 @@ from datetime import datetime -from typing import Optional +from typing import List, Optional from urllib.parse import urlparse -from dendrite.logic.cache import extract_cache +from dendrite.logic.cache.file_cache import FileCache from dendrite.models.scripts import Script -def save_script(code: str, prompt: str, url: str): +def save_script(code: str, prompt: str, url: str, cache: FileCache[Script]): domain = urlparse(url).netloc script = Script( url=url, domain=domain, script=code, created_at=datetime.now().isoformat() ) - extract_cache.ExtractCache.set({"prompt": prompt, "domain": domain}, script) + cache.append({"prompt": prompt, "domain": domain}, script) -def get_script(prompt: str, url: str) -> Optional[Script]: +def get_scripts( + prompt: str, url: str, cache: FileCache[Script] +) -> Optional[List[Script]]: domain = urlparse(url).netloc - return extract_cache.ExtractCache.get({"prompt": prompt, "domain": domain}) + return cache.get({"prompt": prompt, "domain": domain}) diff --git a/dendrite/logic/extract/cached_script.py b/dendrite/logic/extract/cache.py similarity index 53% rename from dendrite/logic/extract/cached_script.py rename to dendrite/logic/extract/cache.py index 8164b29..36aab75 100644 --- a/dendrite/logic/extract/cached_script.py +++ b/dendrite/logic/extract/cache.py @@ -1,24 +1,41 @@ +from datetime import datetime from typing import Any, List, Optional, Tuple from urllib.parse import urlparse from loguru import logger -from dendrite.logic.cache.utils import get_script +from dendrite.logic.cache.file_cache import FileCache from dendrite.logic.code.code_session import execute +from dendrite.logic.config import Config +from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.scripts import Script +def save_script(code: str, prompt: str, url: str, cache: FileCache[Script]): + domain = urlparse(url).netloc + script = Script( + url=url, domain=domain, script=code, created_at=datetime.now().isoformat() + ) + cache.append({"prompt": prompt, "domain": domain}, script) + + +def get_scripts( + prompt: str, url: str, cache: FileCache[Script] +) -> Optional[List[Script]]: + domain = urlparse(url).netloc + return cache.get({"prompt": prompt, "domain": domain}) + + async def get_working_cached_script( - prompt: str, - raw_html: str, - url: str, - return_data_json_schema: Any, + prompt: str, raw_html: str, url: str, return_data_json_schema: Any, config: Config ) -> Optional[Tuple[Script, Any]]: if len(url) == 0: raise Exception("Domain must be specified") - scripts: List[Script] = [get_script(prompt, url) or ...] + scripts = get_scripts(prompt, url, config.extract_cache) + if scripts is None or len(scripts) == 0: + return None logger.debug( f"Found {len(scripts)} scripts in cache | Prompt: {prompt} in domain: {url}" ) @@ -33,9 +50,6 @@ async def get_working_cached_script( ) continue - if len(scripts) == 0: - return None - raise Exception( f"No working script found in cache even though {len(scripts)} scripts were available | Prompt: '{prompt}' in domain: '{url}'" ) diff --git a/dendrite/logic/extract/extract.py b/dendrite/logic/extract/extract.py index 2196bfe..e44664f 100644 --- a/dendrite/logic/extract/extract.py +++ b/dendrite/logic/extract/extract.py @@ -5,9 +5,8 @@ from loguru import logger -from dendrite.logic.cache.utils import get_script from dendrite.logic.config import Config -from dendrite.logic.extract.cached_script import get_working_cached_script +from dendrite.logic.extract.cache import get_scripts, get_working_cached_script from dendrite.logic.extract.extract_agent import ExtractAgent from dendrite.models.dto.cached_extract_dto import CachedExtractDTO from dendrite.models.dto.extract_dto import ExtractDTO @@ -16,11 +15,12 @@ async def get_cached_scripts(dto: CachedExtractDTO, config: Config) -> List[Script]: - script = get_script(dto.prompt, dto.url) - return [script] if script else [] + return get_scripts(dto.prompt, dto.url, config.extract_cache) or [] -async def test_cache(extract_dto: ExtractDTO) -> Optional[ExtractResponse]: +async def test_cache( + extract_dto: ExtractDTO, config: Config +) -> Optional[ExtractResponse]: try: cached_script_res = await get_working_cached_script( @@ -28,6 +28,7 @@ async def test_cache(extract_dto: ExtractDTO) -> Optional[ExtractResponse]: extract_dto.page_information.raw_html, extract_dto.page_information.url, extract_dto.return_data_json_schema, + config, ) if cached_script_res is None: @@ -116,7 +117,7 @@ async def extract(extract_page_dto: ExtractDTO, config: Config) -> ExtractRespon if lock_acquired: return await generate_script(extract_page_dto, lock_manager, config) else: - res = await wait_for_script_generation(extract_page_dto, lock_manager) + res = await wait_for_script_generation(extract_page_dto, lock_manager, config) if res: return res @@ -142,7 +143,7 @@ async def generate_script( async def wait_for_script_generation( - extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager + extract_page_dto: ExtractDTO, lock_manager: InMemoryLockManager, config: Config ) -> Optional[ExtractResponse]: event = await lock_manager.subscribe() logger.info("Waiting for script to be generated") @@ -150,6 +151,6 @@ async def wait_for_script_generation( # If script was created after waiting if notification_received: - res = await test_cache(extract_page_dto) + res = await test_cache(extract_page_dto, config) if res: return res diff --git a/dendrite/logic/extract/extract_agent.py b/dendrite/logic/extract/extract_agent.py index d5baf16..6173351 100644 --- a/dendrite/logic/extract/extract_agent.py +++ b/dendrite/logic/extract/extract_agent.py @@ -6,9 +6,10 @@ from bs4 import BeautifulSoup from dendrite import logger -from dendrite.logic.cache.utils import save_script + from dendrite.logic.config import Config from dendrite.logic.dom.strip import mild_strip +from dendrite.logic.extract.cache import save_script from dendrite.logic.extract.prompts import ( LARGE_HTML_CHAR_TRUNCATE_LEN, create_script_prompt_segmented_html, @@ -168,6 +169,7 @@ async def code_script_from_found_expanded_html_tags( self.generated_script, extract_page_dto.combined_prompt, self.page_information.url, + cache=self.config.extract_cache, ) return result elif isinstance(result, list): diff --git a/dendrite/logic/get_element/cached_selector.py b/dendrite/logic/get_element/cache.py similarity index 84% rename from dendrite/logic/get_element/cached_selector.py rename to dendrite/logic/get_element/cache.py index 07ad105..8cf9965 100644 --- a/dendrite/logic/get_element/cached_selector.py +++ b/dendrite/logic/get_element/cache.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Optional +from typing import List, Optional from urllib.parse import urlparse from dendrite.logic.cache.file_cache import FileCache @@ -8,7 +8,7 @@ async def get_selector_from_cache( url: str, prompt: str, cache: FileCache[Selector] -) -> Optional[Selector]: +) -> Optional[List[Selector]]: netloc = urlparse(url).netloc return cache.get({"netloc": netloc, "prompt": prompt}) @@ -27,4 +27,4 @@ async def add_selector_to_cache( created_at=created_at, ) - cache.set({"netloc": netloc, "prompt": prompt}, selector) + cache.append({"netloc": netloc, "prompt": prompt}, selector) diff --git a/dendrite/logic/get_element/get_element.py b/dendrite/logic/get_element/get_element.py index 58b65af..56447ad 100644 --- a/dendrite/logic/get_element/get_element.py +++ b/dendrite/logic/get_element/get_element.py @@ -6,7 +6,7 @@ from dendrite.logic.config import Config from dendrite.logic.dom.css import check_if_selector_successful, find_css_selector from dendrite.logic.dom.strip import remove_hidden_elements -from dendrite.logic.get_element.cached_selector import ( +from dendrite.logic.get_element.cache import ( add_selector_to_cache, get_selector_from_cache, ) @@ -82,7 +82,7 @@ async def get_cached_selector(dto: CachedSelectorDTO, config: Config) -> List[Se if db_selectors is None: return [] - return [db_selectors] + return db_selectors # async def check_cache( diff --git a/dendrite/remote/__init__.py b/dendrite/remote/__init__.py index 22f7310..e6b3079 100644 --- a/dendrite/remote/__init__.py +++ b/dendrite/remote/__init__.py @@ -3,4 +3,4 @@ __all__ = [ "BrowserbaseConfig", "BrowserlessConfig", -] \ No newline at end of file +] From 3930c3cc52b36274199746f5bebd035a6912b91d Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Mon, 6 Jan 2025 14:33:22 +0100 Subject: [PATCH 14/18] add logger configuration and remove unused cache utility functions --- dendrite/_loggers/d_logger.py | 7 +++++++ dendrite/logic/cache/utils.py | 21 --------------------- dendrite/logic/llm/config.py | 2 +- 3 files changed, 8 insertions(+), 22 deletions(-) create mode 100644 dendrite/_loggers/d_logger.py delete mode 100644 dendrite/logic/cache/utils.py diff --git a/dendrite/_loggers/d_logger.py b/dendrite/_loggers/d_logger.py new file mode 100644 index 0000000..0ff3276 --- /dev/null +++ b/dendrite/_loggers/d_logger.py @@ -0,0 +1,7 @@ +import sys + +from loguru import logger + +logger.remove() +fmt = "{time: HH:mm:ss.SSS} | {level: <8} | {message}" +logger.add(sys.stderr, level="DEBUG", format=fmt) diff --git a/dendrite/logic/cache/utils.py b/dendrite/logic/cache/utils.py deleted file mode 100644 index 6e9a934..0000000 --- a/dendrite/logic/cache/utils.py +++ /dev/null @@ -1,21 +0,0 @@ -from datetime import datetime -from typing import List, Optional -from urllib.parse import urlparse - -from dendrite.logic.cache.file_cache import FileCache -from dendrite.models.scripts import Script - - -def save_script(code: str, prompt: str, url: str, cache: FileCache[Script]): - domain = urlparse(url).netloc - script = Script( - url=url, domain=domain, script=code, created_at=datetime.now().isoformat() - ) - cache.append({"prompt": prompt, "domain": domain}, script) - - -def get_scripts( - prompt: str, url: str, cache: FileCache[Script] -) -> Optional[List[Script]]: - domain = urlparse(url).netloc - return cache.get({"prompt": prompt, "domain": domain}) diff --git a/dendrite/logic/llm/config.py b/dendrite/logic/llm/config.py index 576d4b0..abc35eb 100644 --- a/dendrite/logic/llm/config.py +++ b/dendrite/logic/llm/config.py @@ -19,7 +19,7 @@ "ask_page_agent": LLM( "claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500 ), - "segment_agent": LLM("gpt-4o", temperature=0, max_tokens=1500), + "segment_agent": LLM("claude-3-haiku-20240307", temperature=0, max_tokens=1500), "select_agent": LLM("claude-3-5-sonnet-20241022", temperature=0, max_tokens=1500), "verify_action_agent": LLM( "claude-3-5-sonnet-20241022", temperature=0.3, max_tokens=1500 From 2a8a4215039dc7fcf8034e66345cc058e690f627 Mon Sep 17 00:00:00 2001 From: Arian Hanifi Date: Mon, 6 Jan 2025 14:38:44 +0100 Subject: [PATCH 15/18] remove old stuff in cli --- dendrite/_cli/main.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/dendrite/_cli/main.py b/dendrite/_cli/main.py index 8f86ac5..4dc798b 100644 --- a/dendrite/_cli/main.py +++ b/dendrite/_cli/main.py @@ -21,14 +21,13 @@ def run_playwright_install(): sys.exit(1) -async def setup_auth(url: str, profile_name: str): +async def setup_auth(url: str): try: async with AsyncDendrite() as browser: await browser.setup_auth( url=url, message="Please log in to the website. Once done, press Enter to continue...", ) - print(f"Authentication profile '{profile_name}' has been saved successfully.") except Exception as e: print(f"Error during authentication setup: {e}") sys.exit(1) @@ -42,11 +41,6 @@ def main(): # Add auth-specific arguments parser.add_argument("--url", help="URL to navigate to for authentication") - parser.add_argument( - "--profile", - default="default", - help="Name for the authentication profile (default: 'default')", - ) args = parser.parse_args() @@ -55,7 +49,7 @@ def main(): elif args.command == "auth": if not args.url: parser.error("The --url argument is required for the auth command") - asyncio.run(setup_auth(args.url, args.profile)) + asyncio.run(setup_auth(args.url)) if __name__ == "__main__": From 5b180a6a12101f3a9e44d5dbc669d3d55449fed3 Mon Sep 17 00:00:00 2001 From: charlesmaddock Date: Mon, 6 Jan 2025 16:47:39 +0100 Subject: [PATCH 16/18] Updated readme and fixed small bugs --- .dendrite/cache/extract.json | 30 + .dendrite/cache/get_element.json | 231 +++ .dendrite/cache/storage_state.json | 1151 +++++++++++ .dendrite/test_cache/extract.json | 1 + .dendrite/test_cache/get_element.json | 11 + .dendrite/test_cache/storage_state.json | 1 + README.md | 189 +- dendrite/.dendrite/cache/extract.json | 1 + dendrite/.dendrite/cache/get_element.json | 32 + dendrite/.dendrite/cache/storage_state.json | 1 + dendrite/logic/dom/css.py | 15 +- .../get_element/agents/prompts/__init__.py | 203 +- .../get_element/agents/prompts/segment.prompt | 107 - .../get_element/agents/prompts/select.prompt | 90 - .../logic/get_element/agents/segment_agent.py | 2 +- dendrite/logic/get_element/get_element.py | 5 +- poetry.lock | 1815 ++++++++--------- pyproject.toml | 3 +- test.py | 26 + 19 files changed, 2693 insertions(+), 1221 deletions(-) create mode 100644 .dendrite/cache/extract.json create mode 100644 .dendrite/cache/get_element.json create mode 100644 .dendrite/cache/storage_state.json create mode 100644 .dendrite/test_cache/extract.json create mode 100644 .dendrite/test_cache/get_element.json create mode 100644 .dendrite/test_cache/storage_state.json create mode 100644 dendrite/.dendrite/cache/extract.json create mode 100644 dendrite/.dendrite/cache/get_element.json create mode 100644 dendrite/.dendrite/cache/storage_state.json delete mode 100644 dendrite/logic/get_element/agents/prompts/segment.prompt delete mode 100644 dendrite/logic/get_element/agents/prompts/select.prompt create mode 100644 test.py diff --git a/.dendrite/cache/extract.json b/.dendrite/cache/extract.json new file mode 100644 index 0000000..efd0e39 --- /dev/null +++ b/.dendrite/cache/extract.json @@ -0,0 +1,30 @@ +{ + "506bbd9210aa4b0001c1bb6b746edb0c": [ + { + "url": "https://www.google.com/search?q=hello+world&sca_esv=56c33dbfb6047401&source=hp&ei=4-h7Z6OvN46pwPAPqt6U0Ac&iflsig=AL9hbdgAAAAAZ3v28yMXGMY0IGzLtj6wbRHxrZRtxR7m&ved=0ahUKEwjjrP6yp-GKAxWOFBAIHSovBXoQ4dUDCA4&uact=5&oq=hello+world&gs_lp=Egdnd3Mtd2l6IgtoZWxsbyB3b3JsZEgMUABYAHAAeACQAQCYAQCgAQCqAQC4AQPIAQD4AQGYAgCgAgCYAwCSBwCgBwA&sclient=gws-wiz", + "domain": "www.google.com", + "script": "# Find all search result containers and extract titles and urls\nresults = []\nfor result in soup.find_all('div', class_='yuRUbf'):\n link = result.find('a')\n if link:\n url = link.get('href')\n title = link.find('h3', class_='LC20lb').get_text() if link.find('h3', class_='LC20lb') else None\n if url and title:\n results.append({\n 'url': url,\n 'title': title\n })\nresponse_data = results", + "created_at": "2025-01-06T15:30:40.319245" + }, + { + "url": "https://www.google.com/search?q=hello+world&sca_esv=56c33dbfb6047401&source=hp&ei=OOl7Z5vyBPLGwPAPtLa8-Qs&iflsig=AL9hbdgAAAAAZ3v3SMS1-bHO8XE5tguRZxhUXc5KxEX9&ved=0ahUKEwib7o_bp-GKAxVyIxAIHTQbL78Q4dUDCA4&uact=5&oq=hello+world&gs_lp=Egdnd3Mtd2l6IgtoZWxsbyB3b3JsZEgKUABYAHAAeACQAQCYAQCgAQCqAQC4AQPIAQD4AQGYAgCgAgCYAwCSBwCgBwA&sclient=gws-wiz", + "domain": "www.google.com", + "script": "results = []\nfor result in soup.find_all('div', class_='yuRUbf'):\n link = result.find('a')\n if link:\n url = link.get('href')\n title = link.find('h3', class_='LC20lb')\n if title:\n title = title.get_text(strip=True)\n results.append({\n 'url': url,\n 'title': title\n })\nresponse_data = results", + "created_at": "2025-01-06T15:31:54.390225" + } + ], + "efcae8c2d66eb756a9d08869713c650d": [ + { + "url": "https://dendrite.systems/", + "domain": "dendrite.systems", + "script": "response_data = soup.find('div', {'class': 'chakra-stack css-11upoe7'}).prettify()", + "created_at": "2025-01-06T16:08:19.639139" + }, + { + "url": "https://dendrite.systems/", + "domain": "dendrite.systems", + "script": "response_data = soup.find('div', {'class': 'chakra-stack css-11upoe7'}).prettify()", + "created_at": "2025-01-06T16:19:11.196902" + } + ] +} \ No newline at end of file diff --git a/.dendrite/cache/get_element.json b/.dendrite/cache/get_element.json new file mode 100644 index 0000000..ab287aa --- /dev/null +++ b/.dendrite/cache/get_element.json @@ -0,0 +1,231 @@ +{ + "7b94db3c84c0214bb29f20c70efc4f86": [ + { + "selector": "div[id=\"news-item-0\"] > div:nth-child(3) > div:nth-child(3) > div > noscript > div", + "prompt": "link for all latest private buy and sell postings\n\nThe element should be clickable.", + "url": "https://www.sweclockers.com/", + "netloc": "www.sweclockers.com", + "created_at": "2025-01-06T15:09:51.410885" + } + ], + "47787fc6aa09b7b6445315d032a16e3a": [ + { + "selector": "body > noscript > meta", + "prompt": "Reject all cookies\n\nThe element should be clickable.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:10:26.905612" + }, + { + "selector": "body > noscript > meta", + "prompt": "Reject all cookies\n\nThe element should be clickable.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:11:19.229277" + }, + { + "selector": "body > noscript > meta", + "prompt": "Reject all cookies\n\nThe element should be clickable.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:11:24.921313" + }, + { + "selector": "button[id=\"W0wltc\"]", + "prompt": "Reject all cookies\n\nThe element should be clickable.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:30:08.823683" + } + ], + "c1094c675ffbbcc5a0d1f2c3f353f64e": [ + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:11:30.085245" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:11:35.239066" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:11:39.749348" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:11:44.421790" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:13:22.794447" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:13:27.643231" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:16:30.192815" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:16:34.778220" + }, + { + "selector": "textarea[id=\"APjFqb\"]", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:29:04.780560" + }, + { + "selector": "textarea[id=\"APjFqb\"]", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:30:14.829934" + }, + { + "selector": "textarea[id=\"APjFqb\"]", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:31:25.919497" + } + ], + "f68f83fb21a2d0573cc855ee8508cace": [ + { + "selector": "button[aria-label=\"New\\ mail\"]", + "prompt": "The new email button\n\nThe element should be clickable.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:26:49.927767" + } + ], + "6f7b4a48ea5909af9eca6ab2d0afaa60": [ + { + "selector": "div[aria-label=\"To\"]", + "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Recipient', 'Subject', 'Message']) in this page. Get the field best described as 'Recipient'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:27:01.932492" + } + ], + "2756844c43a183655e870811fe56f9e8": [ + { + "selector": "span.DeHTj", + "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Recipient', 'Subject', 'Message']) in this page. Get the field best described as 'Subject'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:27:14.555811" + } + ], + "b2c1bd5c5ec4557d93d091e581da8d20": [ + { + "selector": "div[aria-label=\"Message\\ body\\,\\ press\\ Alt\\+F10\\ to\\ exit\"]", + "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Recipient', 'Subject', 'Message']) in this page. Get the field best described as 'Message'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:27:26.121823" + } + ], + "e33e95dde2eacd1d3919fda132bd8f5a": [ + { + "selector": "button[id=\"splitButton-r24__primaryActionButton\"]", + "prompt": "The send button\n\nThe element should be clickable.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:27:36.927998" + }, + { + "selector": "button[id=\"splitButton-r1p__primaryActionButton\"]", + "prompt": "The send button\n\nThe element should be clickable.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:30:12.619521" + }, + { + "selector": "button[id=\"splitButton-r1e__primaryActionButton\"]", + "prompt": "The send button\n\nThe element should be clickable.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:43:14.041097" + } + ], + "f186d514ed4f8f795ca083b1cc661b05": [ + { + "selector": "div[aria-label=\"To\"]", + "prompt": "The recipient field\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:28:41.359533" + } + ], + "783d6e8491de54537999dd4a39069f99": [ + { + "selector": "span.DeHTj", + "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Subject', 'Message']) in this page. Get the field best described as 'Subject'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:29:42.113739" + } + ], + "9519aa7364cd93eedbb70dca73a7336e": [ + { + "selector": "div[aria-label=\"Message\\ body\\,\\ press\\ Alt\\+F10\\ to\\ exit\"]", + "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Subject', 'Message']) in this page. Get the field best described as 'Message'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:29:53.726664" + } + ], + "17e46caa6a5819956a7c46d62b65ac0e": [ + { + "selector": "input[aria-label=\"Add\\ a\\ subject\"]", + "prompt": "The subject field\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:40:35.649464" + } + ], + "71d43663082637d3bbbd34d776d3bea4": [ + { + "selector": "div.elementToProof", + "prompt": "The message field\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:40:45.019897" + }, + { + "selector": "div[aria-label=\"Message\\ body\\,\\ press\\ Alt\\+F10\\ to\\ exit\"]", + "prompt": "The message field\n\nMake sure the element can be filled with text.", + "url": "https://outlook.live.com/mail/0/", + "netloc": "outlook.live.com", + "created_at": "2025-01-06T16:42:56.855528" + } + ] +} \ No newline at end of file diff --git a/.dendrite/cache/storage_state.json b/.dendrite/cache/storage_state.json new file mode 100644 index 0000000..626a3a0 --- /dev/null +++ b/.dendrite/cache/storage_state.json @@ -0,0 +1,1151 @@ +{ + "2813eeabfd22ab1b344ad7263486fd4d": [ + { + "origins": [ + { + "origin": "https://outlook.live.com", + "localStorage": [ + { + "name": "olk-dla", + "value": "1736176839937" + }, + { + "name": "olk-WebPushUserSettings", + "value": "[{\"userPreferenceIdentifier\":\"00037fff-8441-f3e2-0000-000000000000_\",\"lastSessionDate\":\"Mon Jan 06 2025\",\"uniqueDaysSessionCount\":1,\"enabled\":null,\"mailEnabled\":null,\"reminderEnabled\":null,\"vipMailEnabled\":null,\"lastCheckboxUnchecked\":null,\"promptCount\":0,\"lastPromptDate\":null}]" + }, + { + "name": "olk-UsersNormalizedThemeImage", + "value": "assets/mail/themes/modern/v2/arcticsolitude/light.jpg" + }, + { + "name": "olk-isTimeZoneCacheAvailable", + "value": "true" + }, + { + "name": "olk-ActionableMessages.ClientConfiguration_960e1055cb51450f0a444f174f48fd4ad9bec92b5f78966f6481c00645c7a8aa", + "value": "{\"data\":{\"organizationConfigs\":{\"ConnectorsEnabled\":true,\"ConnectorsActionableMessagesEnabled\":true,\"OutlookPayEnabled\":false,\"SmtpActionableMessagesEnabled\":true},\"providerConfigs\":[{\"originator\":\"5ED5E1C4-1023-46CD-8D17-F8942D0CD5DD\"},{\"originator\":\"62242E1D-C39A-4F8A-A66E-1D39C0A336B0\"},{\"originator\":\"569a3659-9ebb-4ffc-98c5-7796c9f9d695\"},{\"originator\":\"87ffcf3a-1b34-41ea-88ff-21eaae9870ee\"},{\"originator\":\"6d4f58eb-dcdd-4fe5-b554-82d9305ce7ee\"},{\"originator\":\"6acc43d6-4718-4d95-8f05-42311a7188a7\"}],\"providerConfigsDarkMode\":[{\"originator\":\"5ED5E1C4-1023-46CD-8D17-F8942D0CD5DD\"},{\"originator\":\"62242E1D-C39A-4F8A-A66E-1D39C0A336B0\"},{\"originator\":\"569a3659-9ebb-4ffc-98c5-7796c9f9d695\"},{\"originator\":\"87ffcf3a-1b34-41ea-88ff-21eaae9870ee\"},{\"originator\":\"6d4f58eb-dcdd-4fe5-b554-82d9305ce7ee\"},{\"originator\":\"6acc43d6-4718-4d95-8f05-42311a7188a7\"}],\"themeMappings\":{\"connectors\":{\"teams\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/teams/teams.json\",\"infobar-generic\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/infobar/infobar-generic.json\",\"infobar-alert\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/infobar/infobar-alert.json\",\"txp-demo\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/txp/txp-demo.json\",\"txp-opay\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/txp/txp-opay.json\",\"agendamail\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/agendamail/agendamail.json\",\"compact\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/compact/compact.json\",\"myanalytics\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/myanalytics/myanalytics.json\",\"ram-word\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-word.json\",\"ram-excel\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-excel.json\",\"ram-powerpoint\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-powerpoint.json\",\"reply_at_mentions-word\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-word.json\",\"reply_at_mentions-excel\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-excel.json\",\"reply_at_mentions-powerpoint\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-powerpoint.json\",\"cseo\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/cseo/cseo.json\",\"unifiedgroups\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/unifiedgroups/unifiedgroups.json\",\"cortana\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/cortana/cortana.json\",\"msteams\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/msteams/msteams.json\",\"surveymonkey\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/surveymonkey/surveymonkey.json\",\"officeforms\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/officeforms/officeforms.json\",\"datahygieneengine\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/datahygieneengine/datahygieneengine.json\"}},\"themeMappingsDarkMode\":{\"connectors\":{}},\"hostCapabilities\":{\"AADAuthentication\":\"*\",\"Transaction\":\"*\"},\"clientTimeouts\":{\"default\":{\"AutoInvokedActionTimeoutInMsec\":6000,\"HttpActionTimeoutInMsec\":15000,\"SnackBarTimeoutInMsec\":8000}},\"clientTimeoutsACv2\":{\"default\":{\"AutoInvokedActionTimeoutInMsec\":10000,\"HttpActionTimeoutInMsec\":20000,\"SnackBarTimeoutInMsec\":8000}}},\"state\":1,\"time\":1736176852482}" + }, + { + "name": "olk-bootstrapMailListItemWindowWidth", + "value": "1280" + }, + { + "name": "olk-sla", + "value": "0" + }, + { + "name": "olk-tour-state", + "value": "{\"lastClientActiveDate\":\"Mon, 06 Jan 2025 15:20:52 GMT\",\"tourEnabled\":false,\"tourCollapsed\":false,\"moduleTourStates\":{\"Mail\":{\"tourEnabled\":false,\"tourCollapsed\":false}}}" + }, + { + "name": "olk-sdfp", + "value": "{\"TimeZoneStr\":\"Greenwich Standard Time\",\"FolderPaneBitFlags\":0}" + }, + { + "name": "sessionTracking_00037FFF8441F3E2", + "value": "{\"authenticatedState\":1,\"upn\":\"dendrite_labs@outlook.com\",\"idp\":\"msa\",\"lastActiveTime\":1736176839463}" + }, + { + "name": "olk-MailOwaPreloadStrings", + "value": "[\"https://res.cdn.office.net/owamail/hashed-v1/scripts/../resources/locale/en/owa.AppBoot.m.ce09dc99.json\"]" + }, + { + "name": "olk-UsersNormalizedTheme", + "value": "arcticsolitude" + }, + { + "name": "olk-ActionableMessages.ActionEndpoint_960e1055cb51450f0a444f174f48fd4ad9bec92b5f78966f6481c00645c7a8aa", + "value": "{\"data\":{\"Url\":\"/actionsb2netcore\"},\"state\":2,\"time\":1736176852045,\"successiveCount\":1}" + }, + { + "name": "olk-BootDiagnostics", + "value": "{\"puid\":\"00037FFF8441F3E2\",\"tid\":\"84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa\",\"mbx\":\"00037fff-8441-f3e2-0000-000000000000\",\"prem\":\"0\",\"isCon\":true,\"upn\":\"dendrite_labs@outlook.com\"}" + }, + { + "name": "olk-LogicalRing", + "value": "WW" + }, + { + "name": "olk-mail_conditionalformattingdendrite_labs@outlook.com", + "value": "[]" + }, + { + "name": "olk-UnifiedConsentRequestData", + "value": "{\"id\":\"c147f8be-56d9-452c-b20e-b105df5eba21_ucsisunotice_1\",\"modelType\":\"ucsisunotice\",\"status\":\"read\",\"version\":\"1.14\",\"consentType\":\"prominentnotice\",\"maxPromptsReached\":true,\"consentedBy\":\"user\",\"consentedInSurface\":\"appstart\",\"consentedUTC\":\"2024-10-17T09:59:13.1833406+00:00\",\"values\":null,\"needConsent\":false,\"consentReason\":\"unknown\",\"lastRequestedConsentTimeInMsUTC\":1736176846100}" + }, + { + "name": "olk-bootstrapMailListItemViewSwapSetting", + "value": "true" + }, + { + "name": "olk-EnvDiagnostics", + "value": "{\"fe\":\"BN0PR04CA0083, GV3P280CA0102\",\"be\":\"BN8PR03MB4850\",\"wsver\":\"15.20.8314.18\",\"fost\":\"NAMPRD03\",\"dag\":\"NAMPR03DG090\",\"te\":\"0\"}" + }, + { + "name": "olk-OwaClientId", + "value": "F92C019C4DC04A9A8B76EE06C7E7B683" + }, + { + "name": "olk-slaef", + "value": "-1" + }, + { + "name": "O365Shell_ECS_config", + "value": "{\"9897789ed7282bc5e2a60091cd781bb2b482e27793b0479995f1209675fa3ec3\":{\"expiryTime\":1736263238873,\"value\":{\"OneShell\":{\"UpdatedConsumerAppList\":true,\"M365StartEnabled\":true,\"DisableM365StartIntentsModule\":false,\"default\":true},\"Headers\":{\"ETag\":\"\\\"8M6C3IBLtb8mwT1KNreplkZ/i0rFbbeiyWPkxzwWrg0=\\\"\",\"Expires\":\"Mon, 06 Jan 2025 16:20:38 GMT\",\"CountryCode\":\"SE\",\"StatusCode\":\"200\"},\"ConfigIDs\":{\"OneShell\":\"P-R-1157040-4-8,P-R-1131228-4-17,P-D-1117449-1-4\"}}}}" + }, + { + "name": "sessionTracking_SignedInAccountList", + "value": "[{\"sessionTrackingKey\":\"sessionTracking_00037FFF8441F3E2\",\"lastActiveTime\":1736176839463}]" + }, + { + "name": "olk-dlaef", + "value": "-1" + }, + { + "name": "olk-leftNavAtV2DisplayDate", + "value": "2025-01-06T15:20:40.091Z" + }, + { + "name": "olk-OwaLocale", + "value": "en" + }, + { + "name": "olk-OwaSessionCount", + "value": "1" + }, + { + "name": "olk-undefinedOwaPreloadStrings", + "value": "[\"https://res.cdn.office.net/owamail/hashed-v1/scripts/../resources/locale/en/owa.worker.data.b526d83d.json\"]" + }, + { + "name": "olk-ActionableMessages.ClientConfiguration", + "value": "{\"data\":{\"organizationConfigs\":{\"ConnectorsEnabled\":true,\"ConnectorsActionableMessagesEnabled\":true,\"OutlookPayEnabled\":false,\"SmtpActionableMessagesEnabled\":true},\"providerConfigs\":[{\"originator\":\"5ED5E1C4-1023-46CD-8D17-F8942D0CD5DD\"},{\"originator\":\"62242E1D-C39A-4F8A-A66E-1D39C0A336B0\"},{\"originator\":\"569a3659-9ebb-4ffc-98c5-7796c9f9d695\"},{\"originator\":\"87ffcf3a-1b34-41ea-88ff-21eaae9870ee\"},{\"originator\":\"6d4f58eb-dcdd-4fe5-b554-82d9305ce7ee\"},{\"originator\":\"6acc43d6-4718-4d95-8f05-42311a7188a7\"}],\"providerConfigsDarkMode\":[{\"originator\":\"5ED5E1C4-1023-46CD-8D17-F8942D0CD5DD\"},{\"originator\":\"62242E1D-C39A-4F8A-A66E-1D39C0A336B0\"},{\"originator\":\"569a3659-9ebb-4ffc-98c5-7796c9f9d695\"},{\"originator\":\"87ffcf3a-1b34-41ea-88ff-21eaae9870ee\"},{\"originator\":\"6d4f58eb-dcdd-4fe5-b554-82d9305ce7ee\"},{\"originator\":\"6acc43d6-4718-4d95-8f05-42311a7188a7\"}],\"themeMappings\":{\"connectors\":{\"teams\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/teams/teams.json\",\"infobar-generic\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/infobar/infobar-generic.json\",\"infobar-alert\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/infobar/infobar-alert.json\",\"txp-demo\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/txp/txp-demo.json\",\"txp-opay\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/txp/txp-opay.json\",\"agendamail\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/agendamail/agendamail.json\",\"compact\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/compact/compact.json\",\"myanalytics\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/myanalytics/myanalytics.json\",\"ram-word\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-word.json\",\"ram-excel\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-excel.json\",\"ram-powerpoint\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-powerpoint.json\",\"reply_at_mentions-word\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-word.json\",\"reply_at_mentions-excel\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-excel.json\",\"reply_at_mentions-powerpoint\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-powerpoint.json\",\"cseo\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/cseo/cseo.json\",\"unifiedgroups\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/unifiedgroups/unifiedgroups.json\",\"cortana\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/cortana/cortana.json\",\"msteams\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/msteams/msteams.json\",\"surveymonkey\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/surveymonkey/surveymonkey.json\",\"officeforms\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/officeforms/officeforms.json\",\"datahygieneengine\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/datahygieneengine/datahygieneengine.json\"}},\"themeMappingsDarkMode\":{\"connectors\":{}},\"hostCapabilities\":{\"AADAuthentication\":\"*\",\"Transaction\":\"*\"},\"clientTimeouts\":{\"default\":{\"AutoInvokedActionTimeoutInMsec\":6000,\"HttpActionTimeoutInMsec\":15000,\"SnackBarTimeoutInMsec\":8000}},\"clientTimeoutsACv2\":{\"default\":{\"AutoInvokedActionTimeoutInMsec\":10000,\"HttpActionTimeoutInMsec\":20000,\"SnackBarTimeoutInMsec\":8000}}},\"state\":1,\"time\":1736176852482}" + }, + { + "name": "O365Shell_ThemeInfo_Consumer", + "value": "{\"Id\":\"arcticsolitude\",\"Primary\":\"#0F6CBD\",\"NavBar\":\"transparent\",\"DefaultBackground\":\"transparent\",\"BackgroundImageUrl\":null,\"DefaultText\":\"#242424cc\",\"AppName\":\"#242424\",\"FullBleedImages\":null,\"userPersonalizationAllowed\":true}" + }, + { + "name": "olk-sdmp", + "value": "{\"TimeZoneStr\":\"Greenwich Standard Time\",\"InboxReadingPanePosition\":1,\"IsFocusedInboxOn\":true,\"BootWithConversationView\":true,\"SortResults\":[{\"Path\":{\"__type\":\"PropertyUri:#Exchange\",\"FieldURI\":\"conversation:LastDeliveryOrRenewTime\"},\"Order\":\"Descending\"},{\"Path\":{\"__type\":\"PropertyUri:#Exchange\",\"FieldURI\":\"conversation:LastDeliveryTime\"},\"Order\":\"Descending\"}],\"IsSenderScreeningSettingEnabled\":false}" + }, + { + "name": "olk-ActionableMessages.ActionEndpoint", + "value": "{\"data\":{\"Url\":\"/actionsb2netcore\"},\"state\":2,\"time\":1736176852045,\"successiveCount\":1}" + }, + { + "name": "olk-SmtpActionableMessagesEnabled_dendrite_labs@outlook.com", + "value": "true" + }, + { + "name": "olk-bootFailureCount", + "value": "0" + } + ] + } + ], + "cookies": [ + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/orgexplorer/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/platformapp/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/orgexplorer/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/platformapp/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/orgexplorer/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/platformapp/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/orgexplorer/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/platformapp/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/bookwithme/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/bookwithme/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/bookwithme/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/bookwithme/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/calendar/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/calendar/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/calendar/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/calendar/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/people/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/photos/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/spaces/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/mailb2/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/people/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/photos/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/spaces/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/mailb2/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/people/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/photos/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/spaces/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/mailb2/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/people/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/photos/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/spaces/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/mailb2/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/files/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/files/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/files/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "ConnectorsLtiToken", + "value": "eyJhbGciOiJSUzI1NiIsImtpZCI6IkEzMDVCMkU1Q0ZERjFGQTFBODgyNTU2MzM3NDhCQkNBRTAxNUU5OTIiLCJ0eXAiOiJKV1QiLCJ4NXQiOiJvd1d5NWNfZkg2R29nbFZqTjBpN3l1QVY2WkkifQ.eyJvaWQiOiIwMDAzN2ZmZi04NDQxLWYzZTItMDAwMC0wMDAwMDAwMDAwMDAiLCJwdWlkIjoiMDAwMzdGRkY4NDQxRjNFMiIsInNtdHAiOiJkZW5kcml0ZV9sYWJzQG91dGxvb2suY29tIiwiY2lkIjoiNUQ5OEEwQTY1QjkxNDk4QyIsInZlciI6IkV4Y2hhbmdlLkNhbGxiYWNrLlYyIiwiYXBwaWQiOiI0MGI4NTJkZi05NjkzLTQyMGQtYWE3ZC0xMDA3MmM5Y2UwNzciLCJkZXBsb3ltZW50aWQiOiJodHRwczovL291dGxvb2sub2ZmaWNlMzY1LmNvbS8iLCJ0aWQiOiI4NGRmOWU3Zi1lOWY2LTQwYWYtYjQzNS1hYWFhYWFhYWFhYWEiLCJhY3IiOiIxIiwiYXBwaWRhY3IiOiIwIiwic2NwIjoiQ29ubmVjdG9ycy5NYW5hZ2VtZW50LldlYiIsIm5iZiI6MTczNjE3Njg1MiwiZXhwIjoxNzM2MTc3NzUyLCJpc3MiOiJodHRwczovL291dGxvb2sub2ZmaWNlMzY1LmNvbS8iLCJhdWQiOiJodHRwczovL291dGxvb2sub2ZmaWNlLmNvbSIsImhhcHAiOiJvd2EifQ.H1tXgOQdUGkqTaIbrGxwmBopFXiX21_UV-c2AaflhJxXa6AE2IGRgrZqbc5IhM296hsckXyEUqk7LzRtS0IZcT54DiokmIpmNQQ_8b_4KPy5WwrgtBDJPSwPJekfNusSN-L4CHiOS6oVq8gqQrQ16pjOZOUCrON5pcR1RpeJ6WClEovUAjv69ZGyxm9QMEw0cK6UvSZeDUvmUsMZpzs8spnO-dxGKHf_PmAz065LV3W3x3BwyuRBUzgOPYwE7T_Vr1US6qiQDQjyBYwxnF5IgKCqM-aQku7UJCVEY7-hDqw6YpjkCVvlZRb8QX8dZ9lFTl-N7arnVi5Z2QS46VUQ6w", + "domain": "outlook.live.com", + "path": "/actions/", + "expires": -1, + "httpOnly": false, + "secure": false, + "sameSite": "Lax" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/files/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/mail/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/host/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/meet/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/feed/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/mail/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/host/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/meet/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/feed/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/mail/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/host/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/meet/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/feed/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/mail/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/host/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/meet/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/feed/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "DefaultAnchorMailbox", + "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", + "domain": "outlook.live.com", + "path": "/owa/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "O365Consumer", + "value": "1", + "domain": "outlook.live.com", + "path": "/owa/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SuiteServiceProxyKey", + "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", + "domain": "outlook.live.com", + "path": "/owa/0/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "MicrosoftApplicationsTelemetryDeviceId", + "value": "cab7afd7-5310-423d-adeb-bf6a43c995c1", + "domain": "outlook.live.com", + "path": "/mail/0", + "expires": 1767712852, + "httpOnly": false, + "secure": false, + "sameSite": "Lax" + }, + { + "name": "MicrosoftApplicationsTelemetryFirstLaunchTime", + "value": "2025-01-06T15:20:52.066Z", + "domain": "outlook.live.com", + "path": "/mail/0", + "expires": 1767712852, + "httpOnly": false, + "secure": false, + "sameSite": "Lax" + }, + { + "name": "X-OWA-CANARY", + "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", + "domain": "outlook.live.com", + "path": "/owa/0/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "ClientId", + "value": "F92C019C4DC04A9A8B76EE06C7E7B683", + "domain": "outlook.live.com", + "path": "/", + "expires": 1767712795.734867, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSPBack", + "value": "0", + "domain": ".login.live.com", + "path": "/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "exchangecookie", + "value": "b32957dafaba492f98a9062fe2472cd8", + "domain": "outlook.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "RpsCsrfState.W7_rqj9rYnRPOvCY0n5Lhs6vO06f66uJX8aL2DyFR3U", + "value": "59cdafb6-3962-e10d-510d-e14b1a5bd5c1", + "domain": "outlook.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSCC", + "value": "88.131.152.178-SE", + "domain": ".login.live.com", + "path": "/", + "expires": 1769872798.800552, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "MicrosoftApplicationsTelemetryDeviceId", + "value": "9b74c3ff-d18e-404d-a664-f01ee22c8c9e", + "domain": "login.live.com", + "path": "/", + "expires": 1767712834.773738, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "fptctx2", + "value": "taBcrIH61PuCVH7eNCyH0FWPWMZs3CpAZMKmhMiLe%252bEHNIBLAHqWFybWaDSk%252bsssqnNNUqnhfYAnMHGshIu0%252bjXl9qScCvWinzgWgiL%252bFYOdabJKXklG1D6Hk8yNrTt6GjEx4%252fmDTbsQIseN%252fnFY5WX2XfvhARX640Pg0rSmBBVOsEeU1MUIAY9GEOI4idGM62Vor91JDF3k3y8TLHsw8Tks24iOWupX%252f6NYrU8awu9uRLDBbe%252fzR3wsb3XaXcnundPGCwes7VczQDwgLzTn1wHsbzcJySiR4HaQLJ0HMAFhQbsiOiy9Yw5T7%252bKojDmysP0n4PAwyhyolmr0eha4tw%253d%253d", + "domain": ".live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "Lax" + }, + { + "name": "MSFPC", + "value": "GUID=e82e1b9d7c7c40788ec16bf278e17420&HASH=e82e&LV=202501&V=4&LU=1736176799208", + "domain": "login.live.com", + "path": "/", + "expires": 1767712803.240902, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "PPLState", + "value": "1", + "domain": ".live.com", + "path": "/", + "expires": 1769872846.66053, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "ai_session", + "value": "zALKfrSoOmqE1KVXys7eui|1736176799114|1736176834880", + "domain": "login.live.com", + "path": "/", + "expires": 1736178634, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSPOK", + "value": "$uuid-ed5ca974-dac6-4f3d-a99c-b4c73611ba60$uuid-013dfd04-9ec5-4686-b50d-7ae7e98fd6b4", + "domain": ".login.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSPPre", + "value": "dendrite_labs%40outlook.com%7c5d98a0a65b91498c%7c%7c", + "domain": ".login.live.com", + "path": "/", + "expires": 1769872838.119016, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSPCID", + "value": "5d98a0a65b91498c", + "domain": ".login.live.com", + "path": "/", + "expires": 1769872846.660561, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSPAuth", + "value": "Disabled", + "domain": ".live.com", + "path": "/", + "expires": 1769872838.119111, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSPProf", + "value": "Disabled", + "domain": ".live.com", + "path": "/", + "expires": 1769872838.119162, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "NAP", + "value": "V=1.9&E=1e56&C=Rn7_DxqIlYAE2TP5QPR538qSgaQtg3RHjK_azveARr5BW9RwChqvEQ&W=1", + "domain": ".live.com", + "path": "/", + "expires": 1744842038.165454, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "ANON", + "value": "A=59A764CC7FC76B140B47E186FFFFFFFF&E=1eb0&W=1", + "domain": ".live.com", + "path": "/", + "expires": 1753482038.16544, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "WLSSC", + "value": "EgAuAgMAAAAMgAAAqAABbz75A45XrIQgHJ9K1NwFTgoY4641GLZglltOVU0vAOgYECkIWy2kdzc1Zip4MNzSKeCexmaq7TXRLNnzZzEpobFUU+9PKODnlRlq1ipfdvohueLyjvSSUPPpd/wQpl1liyBQs6iS8bpF9i4S2txA9C8UmRCN0Ucl/xdx4uPhJRUd9sU+LZJ4jGy8mlnWbkTEPVz4KXTZhKdWneAewpgQwrb148LG44sCPx4ia8wjq92Z08LQiYCQb78nYV1KL6CQbuoTAYWYEKHvRLECPdZZrerbB1tj0wnC7Ff1dnOTYmSf23GI4i/ZWmYHwKnBYpD+fcCvqAJogqHeISATgXV1lR0BfgAdAf9/AwDi80GExfR7Z8L0e2cQJwAAChCggBAaAGRlbmRyaXRlX2xhYnNAb3V0bG9vay5jb20AXAAAJmRlbmRyaXRlX2xhYnMlb3V0bG9vay5jb21AcGFzc3BvcnQuY29tAAAAA1VTAAAAAAAABAkCAACPp1VAAAZDAAZkZW5kcmkAAnRlAAAAAAAAAAAAAAAAAAAAAAAAW5FJjF2YoKYAAMX0e2fCm/JnAAAAAAAAAAAAAAAADwA4OC4xMzEuMTUyLjE3OAAFAQAAAAAAAAAAAAAAAAEEAAAAAAAAAAAAAAAAAAAAymcAWWR/tQsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAA", + "domain": ".live.com", + "path": "/", + "expires": 1769872838.119594, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "SDIDC", + "value": "CjRY4d4t5GHxoqC*X1II0hjuG8ZudjtJoQVoSzQ5lLHHK9UfiU1FRSajmJGP3*beMk8YZd5HG4J!aYdcShnQ2rcMWNfMvqDrlHfdC!2c5v5WpcqrzVVeFZA!v9QK!9aIiA$$", + "domain": ".login.live.com", + "path": "/", + "expires": 1769872838.11964, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "JSHP", + "value": "3$dendrite_labs%40outlook.com$dendri$te$$2$0$0$6413239986853085028$0", + "domain": ".login.live.com", + "path": "/", + "expires": 1769872846.660633, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "JSH", + "value": "3$dendrite_labs%40outlook.com$dendri$te$$2$0$0$6413239986853085028$0", + "domain": ".login.live.com", + "path": "/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSPSoftVis", + "value": "@72198325083833620@:@", + "domain": ".login.live.com", + "path": "/", + "expires": 1769872838.119817, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "orgName", + "value": "outlook.com", + "domain": "outlook.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "domainName", + "value": "", + "domain": "outlook.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "LI", + "value": "1:S171tHWqeVrL%2bzkcSS0RqY66hIEoMVwIAST92PQh5pM%3d:H4sIAAAAAAAEAGNkYGBgZIAAViib0QDEk0xJzUspyixJjc9JTCp2yC8tycnPz9ZLzs9lAsoKG5jpGxjqGxkYmSoYmloZGVgZmzOD9BqyAEmWkKLSVACrEe9GWwAAAA==", + "domain": "outlook.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "RPSSecAuth", + "value": "FAB2ARSz06Q8p88COKo8sjogMovqkAnFGw5mAAAMgAAAEBwG%2BWt4GLa6EgnIzOLrZ5QgAdoo5%2BdRJLUJfotKOwHycWU5VOZLlO0y25zEASfc81vdQheA8MLfDPybgor7OlskrfFWKSNB8mMjMeOCIqbjwk9hoDqZ6je93JKsKYAGz/Y7FGEjx3/qDLlw0ZtVN%2BnI5cbACEGr3RypygA/AWSzvHa/xM3Gkj31plaeLP4Nw7qbVoTApbx0x/2LxqO0iX9GRuTf9jhw%2Brq2I1uHjMGNE3j4OlcyHZ5VdcrW5FAl2c/baFlzIzEs20U5maotxpaW5dOGEBkwujb7EmygjmKyEm8ZhBM4W6pbf8NwyWNyvMLpvX4JRPqXifgeMTDoNiApBKxyluzvLj0NnYmlx0Nlz8Ar00R8OzE4pStN2NYuesolbGywp3V/JDZSKWu8woDS8iAA%2B3zgzQX7hZXhb/DBGFkN9TlS5Aiq30ehLzP%2BL6PkwT4%3D", + "domain": ".live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "MH", + "value": "MSFT", + "domain": ".live.com", + "path": "/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "UC", + "value": "e5548faecf0346e9850f44c2d29a398c", + "domain": "outlook.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "Lax" + }, + { + "name": "RoutingKeyCookie", + "value": "v2:H2Q6aEo3iWZU4gQEaoMcda8L4ajaSXKSwSfutOKc3oE%3d:00037fff-8441-f3e2-0000-000000000000@outlook.com", + "domain": "outlook.live.com", + "path": "/", + "expires": 1738768837.442808, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "X-OWA-RedirectHistory", + "value": "AkWK5rsBRfxRrGUu3Qg|AsmKpuYBZuYxrGUu3Qg|AhR7n8MBy48XlWUu3Qg|AhrYdSQBl2znkmUu3Qg", + "domain": "outlook.live.com", + "path": "/", + "expires": 1736198557.444199, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "ShCLSessionID", + "value": "1736176838297_0.2452314684191801", + "domain": "outlook.live.com", + "path": "/", + "expires": -1, + "httpOnly": false, + "secure": false, + "sameSite": "Lax" + }, + { + "name": "SM", + "value": "C", + "domain": ".c.live.com", + "path": "/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "MUID", + "value": "39F11D600E5861AB3DB0080C0F7060A2", + "domain": ".live.com", + "path": "/", + "expires": 1769872839.698497, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "SRM_L", + "value": "39F11D600E5861AB3DB0080C0F7060A2", + "domain": ".c.live.com", + "path": "/", + "expires": 1769872839.698539, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "MR", + "value": "0", + "domain": ".c.live.com", + "path": "/", + "expires": 1736781639.698557, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "ANONCHK", + "value": "1", + "domain": ".c.live.com", + "path": "/", + "expires": 1736177439.698573, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSFPC", + "value": "GUID=e82e1b9d7c7c40788ec16bf278e17420&HASH=e82e&LV=202501&V=4&LU=1736176799208", + "domain": "outlook.live.com", + "path": "/", + "expires": 1767712841.237023, + "httpOnly": false, + "secure": true, + "sameSite": "None" + }, + { + "name": "MSPRequ", + "value": "id=N<=1736176845&co=0", + "domain": ".login.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "uaid", + "value": "2a6c3510907e4d03865788fb8b9e86a6", + "domain": ".login.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "__Host-MSAAUTHP", + "value": "11-M.C555_SN1.0.U.CgT9yBmtfX!MolWa0RNVFEaD3YKIeX97uBbCcpo5LT4zq*!uaVErnmGCQzzi!xgcTB4MGhTP9jOhPn91hkT*sCObALKUkS!4wNJBoAvRrmEfAomk!jFAX4h2QgTxUer88!7*dPl!lUt8CCqZX09MDKKUQTBbGqaCAfLCVHgotLim95NJBYEl9rOxVQTjNMKuXr6kUKqND7Y4zYeXJz26Re5JbqYkSQnUgMij2Uws6NH9EQ3rZer08*TwAuFZB6E8tGxKDdoqe945ER0Uo5ed!Nf7SY77qcjO!GzwHHxo35TB", + "domain": "login.live.com", + "path": "/", + "expires": 1769872846.660508, + "httpOnly": true, + "secure": true, + "sameSite": "None" + }, + { + "name": "OParams", + "value": "11O.DggSuOYHeYc8yw53azHyaYZtpkLUm1ENd0gc4tee0dOyzVqzFGCsuBUquUDeQZyiKl!3LcgEg9MmaakouTCa5SvYmB80LT7S1Wb6eIavH0SDk61YFlUky0P6BUMOzfO3GUjuteSGh*jl6i4BZ1C6EjaGQv7o2DBFd6FIfUiP5L7JK72Gfoffx6eHytDXwfhiMB3HMadsDkCx6ENaMHb4z!0F2XvFMPTWIHsIQxGByeMg79tEWVPOnZiUV728NYJUGs6roGZLpBApKii9KgaKrOXB4lnLoGsrGbyfVjZSoatrJ7F*PLVui!KS3R2l53vCYQ9lZoo2vdgSUPVFY3rxi578nPvpB87KohWX5vQZ6MQ5UKm8EfM3shhopCS2qUWtJz*4Qn44c5oFRYxIHjO6a*SQ3DmUNbXcpeZtOTinX8misorsDrCSCFb4AQpCXUIDKdyq9PAqnNv*qmNQDIMCNujcygHdvYBZVeremBuvZsl4uH*qd1a!h!c68en8zwb6wEY2jMN8SZ1ZtcwvRdcVD1W12uwlPeCmUEoP*qXLisHMdSDVBmoHA*M7MPO4Wpy9ghpJttnd*8DubMwgp6lPvmGWPPIgn*SVKYRHUQlgOYBJeowzFI!6LTn7xLo7keRsToROl!whFoyR9!ulLI60SiG!fMoOJdKdlbtsehv9KDz5xGIS8HIqxgbLfNl0mgEiifAgsyP7bBokxc1n8yQcHYv5iT1g7sb!KpNdCeYmYw!uwtWyoIYFQI5CjI9d96pBmwRMvMYnEEoR!3c!GnK4yL8OmNzpUNbEL!JM*XHf0idcnxY3KeXaKtOCylAJl7QptU9pHw18NghE5hRAnocry3pE!UcjaPOhuUMLCnuZO4BcYfTBfNpE7xRlFP0Yqga6N1Q5bCkaVAzza3UL7Fox6XW1Ug*F!gHzQa2dmJDwSAutFNGR2uoWWrcKV5VE4xx5fTFvHleaDBGPe1KYXv94o0dDcFYLcxX6x7jq!vJyvb0cu7ir3hZ*9bwLvRhnr175TXmcSJpCagbsWFYQbLnmkE3uwfGazBeRc6FMtKtYrEgEEQ*35ZRczJb*uyL8pbIhplqm*3Dof4iyB!mXFpGG8B0F3a!rYQM37CLHeHGCRsE*ZCPsyoNvShJxStyB9Pr5CD3jp7tpx6HD0eBDuq!UMhj15mRpjMLhy1XnMeaGwITvyR3!ZmEur4*xslqykbvXHI4nTIis!4224KY1zYq88mqYDSJMeqoeJpijzhLHZcZrCI6VGivr4O0gJ35y!Tdrg6qfdUtd3ekUGA9oUcmHcWDFf2hWqFRU9Fxlw8nRDZ7rp!fWb9eJsS1sh7ml8YssvavXK6!6Po6uLJLVC3ockPWAyOnIj69yrA!B1w4D5LJOildumPYv7XFaVN9g23sJNj*Xeqz*WJAPSOR7wt3SJfKaNeds0SvszwCWM7dsr95SXAv4Kyxx8m01CtQUmMcowdHQTkJb9t8fnK0JlAC*Fm!vhcBItBaJ!pU9T!Gpa6gyxa5gimkmRu2q2ExlvT0zYWSwH*jlWlUlsE3P8KV*7EmqH3nlHRwEGOu3jWXeOmxVVftCKG!aSrYQiTUm2nnV7BJrI!NH2RtefbQP6vYleQA8GsyZx9KfQRdcY*Cv9ygWxs7*LlC2b3qFObE!H2X9owgmGfe7bSxZ678n2OQoo39BBEJmv5hoLGdlhv7Q5mUoYotzt6EHIZM3XFcIRkT83PmMzhedhnY6mORKxeEuwyDdvWs2mQLWa7qtRC0eDOhELgfaRDfX2DAm9QoUoUUjFXdzVl3aTWpOFJUPhXcjKTioN7Rm7TFxmpbLNe*TrWBT7zBZyI9X*jXKYvmS!KkpQ4gTG9Tp3wt8RP8R3n8uFiu6S3DnNRi64Sb3nsKbOaUig3a4!OzdzI1plfH0CLBt7uk6K1YPLaBuFCvfB38psooGNq!TAfsPnvBu7VAEsC6BCTfDyPrZNZpwXwLarTKMle3HrwHdrxJtXsWz!GZ*mxYaKxm4xWHkxP6fivGgFOUAsgx8PApfpIQI2rJDghLYNHwMqYWgpnrHp0XVAOCE8qxEtJX0elaxwJmcR595DGrcqc6Jm3n!H3LR!*Yj*KrZqkQLgbxvk4GdKqPdVNAowY7lexGi2hmDcdkdUh96Qk3nWJo034wEMMl!RxuDHPYVSH3lrEd3E6TgreaopYmyDHIl3hAgHqjrCNMiuuhBkxJQ96V5Uk5KSN*VwBhgb0nAaSwCsOmPxTvwBKZae*UxN62ezH98gwJOyKWsjcxXUuNl6J1kDwjnOd5PaGDsXkUG2zaeDN3KIdzluYvNBylK8RDUDlwLLlRjDLyrYZO2OMZlzOJkzAxFjL8JP*r2yNWUZ9xyzGG88EYxWuDQhy6dcNwKIawm8hrz1QqLVmj!*KhonfTzvgme1kWo12vcNCdEjcphBSMdREAfw6rpD8VVKkuCuTsLR6176HcGiD3WrMLJpdZZlHS6BVwb3d2AqtU4Z0fhjzmC3QXAPzpV4JT4kxnj23iu1H47jznCi2R7FTv!QcDzq1LEG9nEVQ*M!MDmZIWUgnLb9gglIvhe3MlgUBA$", + "domain": ".login.live.com", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": true, + "sameSite": "None" + } + ] + } + ] +} \ No newline at end of file diff --git a/.dendrite/test_cache/extract.json b/.dendrite/test_cache/extract.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.dendrite/test_cache/extract.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/.dendrite/test_cache/get_element.json b/.dendrite/test_cache/get_element.json new file mode 100644 index 0000000..8c17f56 --- /dev/null +++ b/.dendrite/test_cache/get_element.json @@ -0,0 +1,11 @@ +{ + "7b94db3c84c0214bb29f20c70efc4f86": [ + { + "selector": "div[id=\"news-item-0\"] > div:nth-child(3) > div:nth-child(3) > div > noscript > div", + "prompt": "link for all latest private buy and sell postings\n\nThe element should be clickable.", + "url": "https://www.sweclockers.com/", + "netloc": "www.sweclockers.com", + "created_at": "2025-01-06T15:09:16.487844" + } + ] +} \ No newline at end of file diff --git a/.dendrite/test_cache/storage_state.json b/.dendrite/test_cache/storage_state.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.dendrite/test_cache/storage_state.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/README.md b/README.md index 55a888b..2d885d4 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,11 @@ +> **Notice:** The Dendrite SDK is not under active development from us. However, the project will remain fully open source so that you and others can learn from and build upon our work. Feel free to fork, study, or adapt this code for your own projects as you wish – reach out to us on Discord if you have questions! +

    Dendrite Homepage Docs Discord

    -
    -

    🎉We are going open source🎉

    -

    - Let us know if you're interested in contributing! We're working on integrating the core logic for getting elements and extraction into the sdk! -

    -
    - ## What is Dendrite? #### Dendrite is a framework that makes it easy for web AI agents to browse the internet just like humans do. Use Dendrite to: @@ -27,30 +22,44 @@ With Dendrite, it's easy to create web interaction tools for your agent. ```python -from dendrite import Dendrite +from dendrite import AsyncDendrite + -def send_email(): - client = Dendrite(auth="outlook.live.com") +async def send_email(to, subject, message): + client = AsyncDendrite(auth="outlook.live.com") # Navigate - client.goto( - "https://outlook.live.com/mail/0/", - expected_page="An email inbox" + await client.goto( + "https://outlook.live.com/mail/0/", expected_page="An email inbox" ) # Create new email and populate fields - client.click("The new email button") - client.fill_fields({ - "Recipient": to, - "Subject": subject, - "Message": message - }) + await client.click("The new email button") + await client.fill("The recipient field", to) + await client.press("Enter") + await client.fill("The subject field", subject) + await client.fill("The message field", message) # Send email - client.click("The send button") + await client.press("Enter", hold_cmd=True) + + +if __name__ == "__main__": + import asyncio + + asyncio.run(send_email("test@example.com", "Hello", "This is a test email")) + ``` -To authenticate you'll need to use our Chrome Extension **Dendrite Vault**, you can download it [here](https://chromewebstore.google.com/detail/dendrite-vault/faflkoombjlhkgieldilpijjnblgabnn). Read more about authentication [in our docs](https://docs.dendrite.systems/examples/authentication-instagram). +To authenticate on outlook, run the command below: + +```bash +dendrite auth --url outlook.live.com +``` + +A browser will open and you'll be able to login. After you've logged in, press enter in your terminal to save the cookies locally, so that they can be used in your code. + +Read more about authentication [in our docs](https://docs.dendrite.systems/examples/authentication). ## Quickstart @@ -65,75 +74,107 @@ Initialize the Dendrite client and start doing web interactions without boilerpl [Get your API key here](https://dendrite.systems/app) ```python -from dendrite import Dendrite +from dendrite import AsyncDendrite + +async def main(): + client = AsyncDendrite(dendrite_api_key="sk...") -client = Dendrite(dendrite_api_key="sk...") + await client.goto("https://google.com") + await client.fill("Search field", "Hello world") + await client.press("Enter") -client.goto("https://google.com") -client.fill("Search field", "Hello world") -client.press("Enter") +if __name__ == "__main__": + import asyncio + asyncio.run(main()) ``` In the example above, we simply go to Google, populate the search field with "Hello world" and simulate a keypress on Enter. It's a simple example that starts to explore the endless possibilities with Dendrite. Now you can create tools for your agents that have access to the full web without depending on APIs. -## More powerful examples +## More Examples -Now, let's have some fun. Earlier we showed you a simple send_email example. And sending emails is cool, but if that's all our agent can do it kind of sucks. So let's create two cooler examples. +### Get any page as markdown + +This is a simple example of how to get any page as markdown. + +```python +from dendrite import AsyncDendrite +from dotenv import load_dotenv + +async def main(): + browser = AsyncDendrite() + + await browser.goto("https://dendrite.systems") + await browser.wait_for("the page to load") + + # Get the entire page as markdown + md = await browser.markdown() + print(md) + print("=" * 200) + + # Only get a certain part of the page as markdown + data_extraction_md = await browser.markdown("the part about data extraction") + print(data_extraction_md) + +if __name__ == "__main__": + import asyncio + asyncio.run(main()) +``` + +### Extract Data from Google Analytics + +Here's how to get the amount of monthly visitors from Google Analytics using the `extract` function: + +```python +async def get_visitor_count() -> int: + client = AsyncDendrite(auth="analytics.google.com") + + await client.goto( + "https://analytics.google.com/analytics/web", + expected_page="Google Analytics dashboard" + ) + + # The Dendrite extract agent will create a web script that is cached + # and reused. It will self-heal when the website updates + visitor_count = await client.extract("The amount of visitors this month", int) + return visitor_count +``` ### Download Bank Transactions -First up, a tool that allows our AI agent to download our bank's monthly transactions so that they can be analyzed and compiled into a report that can be sent to stakeholders with `send_email`. +Here's tool that allows our AI agent to download our bank's monthly transactions so that they can be analyzed and compiled into a report. ```python -from dendrite import Dendrite +from dendrite import AsyncDendrite -def get_transactions() -> str: - client = Dendrite(auth="mercury.com") +async def get_transactions() -> str: + client = AsyncDendrite(auth="mercury.com") # Navigate and wait for loading - client.goto( + await client.goto( "https://app.mercury.com/transactions", expected_page="Dashboard with transactions" ) - client.wait_for("The transactions to finish loading") + await client.wait_for("The transactions to finish loading") # Modify filters - client.click("The 'add filter' button") - client.click("The 'show transactions for' dropdown") - client.click("The 'this month' option") + await client.click("The 'add filter' button") + await client.click("The 'show transactions for' dropdown") + await client.click("The 'this month' option") # Download file - client.click("The 'export filtered' button") - transactions = client.get_download() + await client.click("The 'export filtered' button") + transactions = await client.get_download() # Save file locally path = "files/transactions.xlsx" - transactions.save_as(path) + await transactions.save_as(path) return path -def analyze_transactions(path: str): - ... +async def analyze_transactions(path: str): + ... # Analyze the transactions with LLM of our choice ``` -### Extract Google Analytics - -Finally, it would be cool if we could add the amount of monthly visitors from Google Analytics to our report. We can do that by using the `extract` function: - -```python -def get_visitor_count() -> int: - client = Dendrite(auth="analytics.google.com") - - client.goto( - "https://analytics.google.com/analytics/web", - expected_page="Google Analytics dashboard" - ) - - # The Dendrite extract agent will create a web script that is cached - # and reused. It will self-heal when the website updates - visitor_count = client.extract("The amount of visitors this month", int) - return visitor_count -``` ## Documentation @@ -145,7 +186,7 @@ def get_visitor_count() -> int: When you want to scale up your AI agents, we support using browsers hosted by Browserbase. This way you can run many agents in parallel without having to worry about the infrastructure. -To start using Browserbase just swap out the `Dendrite` class with `DendriteRemoteBrowser` and add your Browserbase API key and project id, either in the code or in a `.env` file like this: +To start using Browserbase just swap out the `AsyncDendrite` class with `AsyncDendriteRemoteBrowser` and add your Browserbase API key and project id, either in the code or in a `.env` file like this: ```bash # ... previous keys @@ -154,17 +195,19 @@ BROWSERBASE_PROJECT_ID= ``` ```python -# from dendrite import Dendrite -from dendrite import DendriteRemoteBrowser - -... - -# client = Dendrite(...) -client = DendriteRemoteBrowser( - # Use interchangeably with the Dendrite class - browserbase_api_key="...", # or specify the browsebase keys in the .env file - browserbase_project_id="..." -) +# from dendrite import AsyncDendrite +from dendrite import AsyncDendriteRemoteBrowser + +async def main(): + # client = AsyncDendrite(...) + client = AsyncDendriteRemoteBrowser( + # Use interchangeably with the AsyncDendrite class + browserbase_api_key="...", # or specify the browsebase keys in the .env file + browserbase_project_id="..." + ) + ... -... +if __name__ == "__main__": + import asyncio + asyncio.run(main()) ``` diff --git a/dendrite/.dendrite/cache/extract.json b/dendrite/.dendrite/cache/extract.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/dendrite/.dendrite/cache/extract.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/dendrite/.dendrite/cache/get_element.json b/dendrite/.dendrite/cache/get_element.json new file mode 100644 index 0000000..91a7478 --- /dev/null +++ b/dendrite/.dendrite/cache/get_element.json @@ -0,0 +1,32 @@ +{ + "c1094c675ffbbcc5a0d1f2c3f353f64e": [ + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:20:12.336490" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:20:16.831358" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:20:20.718020" + }, + { + "selector": "body > noscript > meta", + "prompt": "Search input field\n\nMake sure the element can be filled with text.", + "url": "https://www.google.com/", + "netloc": "www.google.com", + "created_at": "2025-01-06T15:20:24.813117" + } + ] +} \ No newline at end of file diff --git a/dendrite/.dendrite/cache/storage_state.json b/dendrite/.dendrite/cache/storage_state.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/dendrite/.dendrite/cache/storage_state.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/dendrite/logic/dom/css.py b/dendrite/logic/dom/css.py index 7ae582c..8555df4 100644 --- a/dendrite/logic/dom/css.py +++ b/dendrite/logic/dom/css.py @@ -5,8 +5,19 @@ def find_css_selector(ele: Tag, soup: BeautifulSoup) -> str: - if ele is None: - return "" + logger.debug(f"Finding selector for element: {ele.name} with attrs: {ele.attrs}") + + # Add this debug block + final_selector = "" # Track the selector being built + matches = [] # Track matching elements + + def debug_selector(selector: str) -> None: + nonlocal matches + try: + matches = soup.select(selector) + logger.debug(f"Selector '{selector}' matched {len(matches)} elements") + except Exception as e: + logger.error(f"Invalid selector '{selector}': {e}") # Check for inherently unique elements if ele.name in ["html", "head", "body"]: diff --git a/dendrite/logic/get_element/agents/prompts/__init__.py b/dendrite/logic/get_element/agents/prompts/__init__.py index ef366ac..b6ffe51 100644 --- a/dendrite/logic/get_element/agents/prompts/__init__.py +++ b/dendrite/logic/get_element/agents/prompts/__init__.py @@ -1,8 +1,199 @@ -def load_prompt(prompt_path: str) -> str: - with open(prompt_path, "r") as f: - prompt = f.read() - return prompt +SEGMENT_PROMPT = """You are an agent that is given the task to find candidate elements that match the element that the user is looking for. You will get multiple segments of the html of the page and a description of the element that the user is looking for. +The description can be the text that the element contains, the type of element. You might get both short and long descriptions. +Don't only look for the exact match of the text. +Look at aria-label if there are any as they are helpful in identifying the elements. -SEGMENT_PROMPT = load_prompt("dendrite/logic/get_element/agents/prompts/segment.prompt") -SELECT_PROMPT = load_prompt("dendrite/logic/get_element/agents/prompts/segment.prompt") +You will get the information in the following format: + + + DESCRIPTION + + + + HTML CONTENT + +... + + HTML CONTENT + + +Each element will have an attribute called d-id which you should refer to if you can find the elements that the user is looking for. There might be multiple elements that are fit the user's request, if so include multiple d_id:s. +If you've selected an element you should NOT select another element that is a child of the element you've selected. +Be sure to include a reason for why you selected the elements that you did. Think step by step, what made you choose this element over the others. +Your response should include 2-3 sentences of reasoning and a code block containing json including the backticks, the reason text is just a placeholder. Always include a sentence of reasoning in the output: + +```json +{ + "reason": , + "d_id": ["125292", "9541ad"], + "status": "success" +} +``` + +If no element seems to match the user's request, or you think the page is still loading, output the following with 2-3 sentences of reasoning in the output: + +```json +{ + "reason": , + "status": "failed" or "loading" +} +``` + +Here are some examples to help you understand the task (your response is the content under "Assistant:"): + +Example 1: + +USER: Can you get the d_id of the element that matches this description? + + + pull requests count + + + +
  • + + + Pull requests + + + 14 + + +
    • +     + +ASSISTANT: + +```json +{ + "reason": "I selected this element because it has the class Counter and is a number next to the pull requests text.", + "d_id": ["235512"], + "status": "success" +} +``` + +Example 2: + +USER: Can you get the d_id of the element that matches this description? + + + search bar + + + +
    or tags or their content in your response.""" + +SELECT_PROMPT = """You are a web scraping agent who is an expert at selecting element(s) that the user is asking for. + +You will get the information in the following format: + + + DESCRIPTION + + +```html +HTML CONTENT +``` + +Try to select a single element that you think is the best match for the user's request. The element should be as small as possible while still containing the information that the user is looking for. If there are wrappers select the element inside. Be sure to include a reason for why you selected the element that you did. +To select an element you should refer to the d-id attribute which is a unique identifier for each element. + +Your response should be in the following format, including the backticks. Do all your reasoning in the `reason` field, only output the json: + +```json +{ + "reason": "After looking at the HTML it is clear that '98jorq3' is the correct element since is contains the text 'Hello World' which is exactly what the user asked for.", + "d_id": ["98jorq3"], + "status": "success" +} +``` + +If the requested element doesn't seem to be available on the page, that's OK. Return the following format, including the backticks: + +```json +{ + "reason": "This page doesn't seem to contain any link for a 'Github repository' as requested. The page has had a couple of seconds to load too and there are links for twitter and facebook, but no github. So, it's impossible to find the requested element on this page.", + "status": "impossible" +} +``` + +A page could still be loading, if this is the case you should return the following format, including the backticks: + +```json +{ + "reason": "Since the requested element is missing and the page only loaded in 2 seconds ago, I believe the page is still loading. Let's wait for the page to load and try again.", + "status": "loading" +} +``` + +Here is an example to help you understand how to select the best element: + +USER: + + pull requests count next to commits count + + +```html + + + ... +
  • + + + Commits + + + 24 + + +
    • +
  • + + + Pull requests + + + 14 + + +
    • + ... + + +``` + +ASSISTANT: +```json +{ + "reason": "This is tricky, there are a few elements that could match the user's request (s8yy81 and 781faa), however I selected the element with the d-id 's8yy81' because the span a class Counter and contains a number and is next to a span with the text pull requests.", + "d_id": ["s8yy81"], + "status": "success" +} +``` + +IMPORTANT! +Your reasoning must be limited to 3-4 sentences. +""" diff --git a/dendrite/logic/get_element/agents/prompts/segment.prompt b/dendrite/logic/get_element/agents/prompts/segment.prompt deleted file mode 100644 index eb84be1..0000000 --- a/dendrite/logic/get_element/agents/prompts/segment.prompt +++ /dev/null @@ -1,107 +0,0 @@ -You are an agent that is given the task to find candidate elements that match the element that the user is looking for. You will get multiple segments of the html of the page and a description of the element that the user is looking for. -The description can be the text that the element contains, the type of element. You might get both short and long descriptions. -Don't only look for the exact match of the text. - -Look at aria-label if there are any as they are helpful in identifying the elements. - -You will get the information in the following format: - - - DESCRIPTION - - - - HTML CONTENT - -... - - HTML CONTENT - - -Each element will have an attribute called d-id which you should refer to if you can find the elements that the user is looking for. There might be multiple elements that are fit the user's request, if so include multiple d_id:s. -If you've selected an element you should NOT select another element that is a child of the element you've selected. -Be sure to include a reason for why you selected the elements that you did. Think step by step, what made you choose this element over the others. -Your response should include 2-3 sentences of reasoning and a code block containing json including the backticks, the reason text is just a placeholder. Always include a sentence of reasoning in the output: - -```json -{ - "reason": , - "d_id": ["125292", "9541ad"], - "status": "success" -} -``` - -If no element seems to match the user's request, or you think the page is still loading, output the following with 2-3 sentences of reasoning in the output: - -```json -{ - "reason": , - "status": "failed" or "loading" -} -``` - -Here are some examples to help you understand the task (your response is the content under "Assistant:"): - -Example 1: - -USER: Can you get the d_id of the element that matches this description? - - - pull requests count - - - -
  • - - - Pull requests - - - 14 - - -
    • -     - -ASSISTANT: - -```json -{ - "reason": "I selected this element because it has the class Counter and is a number next to the pull requests text.", - "d_id": ["235512"], - "status": "success" -} -``` - -Example 2: - -USER: Can you get the d_id of the element that matches this description? - - - search bar - - - -
    or tags or their content in your response. \ No newline at end of file diff --git a/dendrite/logic/get_element/agents/prompts/select.prompt b/dendrite/logic/get_element/agents/prompts/select.prompt deleted file mode 100644 index 339d328..0000000 --- a/dendrite/logic/get_element/agents/prompts/select.prompt +++ /dev/null @@ -1,90 +0,0 @@ -You are a web scraping agent who is an expert at selecting element(s) that the user is asking for. - -You will get the information in the following format: - - - DESCRIPTION - - -```html -HTML CONTENT -``` - -Try to select a single element that you think is the best match for the user's request. The element should be as small as possible while still containing the information that the user is looking for. If there are wrappers select the element inside. Be sure to include a reason for why you selected the element that you did. -To select an element you should refer to the d-id attribute which is a unique identifier for each element. - -Your response should be in the following format, including the backticks. Do all your reasoning in the `reason` field, only output the json: - -```json -{ - "reason": "After looking at the HTML it is clear that '98jorq3' is the correct element since is contains the text 'Hello World' which is exactly what the user asked for.", - "d_ids": ["98jorq3"], - "status": "success" -} -``` - -If the requested element doesn't seem to be available on the page, that's OK. Return the following format, including the backticks: - -```json -{ - "reason": "This page doesn't seem to contain any link for a 'Github repository' as requested. The page has had a couple of seconds to load too and there are links for twitter and facebook, but no github. So, it's impossible to find the requested element on this page.", - "status": "impossible" -} -``` - -A page could still be loading, if this is the case you should return the following format, including the backticks: - -```json -{ - "reason": "Since the requested element is missing and the page only loaded in 2 seconds ago, I believe the page is still loading. Let's wait for the page to load and try again.", - "status": "loading" -} -``` - -Here is an example to help you understand how to select the best element: - -USER: - - pull requests count next to commits count - - -```html - - - ... -
  • - - - Commits - - - 24 - - -
    • -
  • - - - Pull requests - - - 14 - - -
    • - ... - - -``` - -ASSISTANT: -```json -{ - "reason": "This is tricky, there are a few elements that could match the user's request (s8yy81 and 781faa), however I selected the element with the d-id 's8yy81' because the span a class Counter and contains a number and is next to a span with the text pull requests.", - "d_ids": ["s8yy81"], - "status": "success" -} -``` - -IMPORTANT! -Your reasoning must be limited to 3-4 sentences. diff --git a/dendrite/logic/get_element/agents/segment_agent.py b/dendrite/logic/get_element/agents/segment_agent.py index 574c821..e03f3fd 100644 --- a/dendrite/logic/get_element/agents/segment_agent.py +++ b/dendrite/logic/get_element/agents/segment_agent.py @@ -92,7 +92,7 @@ async def extract_relevant_d_ids( f"""###### SEGMENT ######\n\n{segment}\n\n###### SEGMENT END ######\n\n""" ) - message += f"Can you get the d_ids of the elements that match the following description:\n\n{prompt} element\n\nIf you've selected an element you should NOT select another element that is a child of the element you've selected. It is important that you follow this." + message += f"Can you get the d_id of the elements that match the following description:\n\n{prompt} element\n\nIf you've selected an element you should NOT select another element that is a child of the element you've selected. It is important that you follow this." message += """\nOutput how you think. Think step by step. if there are multiple candidate elements return all of them. Don't make up d-id for elements if they are not present/don't match the description. Limit your reasoning to 2-3 sentences\nOnly include the json block – don't output an array, only ONE object.""" max_retries = 3 diff --git a/dendrite/logic/get_element/get_element.py b/dendrite/logic/get_element/get_element.py index 56447ad..8e28ec7 100644 --- a/dendrite/logic/get_element/get_element.py +++ b/dendrite/logic/get_element/get_element.py @@ -47,7 +47,10 @@ async def get_new_element( if interactable.dendrite_id is None: interactable.status = "failed" interactable.reason = "No d-id found returned from agent" - tag = soup.find(attrs={"d-id": interactable.dendrite_id}) + print(interactable.dendrite_id) + tag = soup_without_hidden_elements.find( + attrs={"d-id": interactable.dendrite_id} + ) if isinstance(tag, Tag): selector = find_css_selector(tag, soup) cache = config.element_cache diff --git a/poetry.lock b/poetry.lock index d1c159b..82b61ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,137 +1,123 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" -version = "2.4.3" +version = "2.4.4" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.8" files = [ - {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, - {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, + {file = "aiohappyeyeballs-2.4.4-py3-none-any.whl", hash = "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"}, + {file = "aiohappyeyeballs-2.4.4.tar.gz", hash = "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"}, ] [[package]] name = "aiohttp" -version = "3.10.10" +version = "3.11.11" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be7443669ae9c016b71f402e43208e13ddf00912f47f623ee5994e12fc7d4b3f"}, - {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7b06b7843929e41a94ea09eb1ce3927865387e3e23ebe108e0d0d09b08d25be9"}, - {file = "aiohttp-3.10.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:333cf6cf8e65f6a1e06e9eb3e643a0c515bb850d470902274239fea02033e9a8"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:274cfa632350225ce3fdeb318c23b4a10ec25c0e2c880eff951a3842cf358ac1"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9e5e4a85bdb56d224f412d9c98ae4cbd032cc4f3161818f692cd81766eee65a"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b606353da03edcc71130b52388d25f9a30a126e04caef1fd637e31683033abd"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab5a5a0c7a7991d90446a198689c0535be89bbd6b410a1f9a66688f0880ec026"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:578a4b875af3e0daaf1ac6fa983d93e0bbfec3ead753b6d6f33d467100cdc67b"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8105fd8a890df77b76dd3054cddf01a879fc13e8af576805d667e0fa0224c35d"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3bcd391d083f636c06a68715e69467963d1f9600f85ef556ea82e9ef25f043f7"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fbc6264158392bad9df19537e872d476f7c57adf718944cc1e4495cbabf38e2a"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e48d5021a84d341bcaf95c8460b152cfbad770d28e5fe14a768988c461b821bc"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2609e9ab08474702cc67b7702dbb8a80e392c54613ebe80db7e8dbdb79837c68"}, - {file = "aiohttp-3.10.10-cp310-cp310-win32.whl", hash = "sha256:84afcdea18eda514c25bc68b9af2a2b1adea7c08899175a51fe7c4fb6d551257"}, - {file = "aiohttp-3.10.10-cp310-cp310-win_amd64.whl", hash = "sha256:9c72109213eb9d3874f7ac8c0c5fa90e072d678e117d9061c06e30c85b4cf0e6"}, - {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c30a0eafc89d28e7f959281b58198a9fa5e99405f716c0289b7892ca345fe45f"}, - {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:258c5dd01afc10015866114e210fb7365f0d02d9d059c3c3415382ab633fcbcb"}, - {file = "aiohttp-3.10.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:15ecd889a709b0080f02721255b3f80bb261c2293d3c748151274dfea93ac871"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3935f82f6f4a3820270842e90456ebad3af15810cf65932bd24da4463bc0a4c"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:413251f6fcf552a33c981c4709a6bba37b12710982fec8e558ae944bfb2abd38"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1720b4f14c78a3089562b8875b53e36b51c97c51adc53325a69b79b4b48ebcb"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:679abe5d3858b33c2cf74faec299fda60ea9de62916e8b67e625d65bf069a3b7"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79019094f87c9fb44f8d769e41dbb664d6e8fcfd62f665ccce36762deaa0e911"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2fb38c2ed905a2582948e2de560675e9dfbee94c6d5ccdb1301c6d0a5bf092"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a3f00003de6eba42d6e94fabb4125600d6e484846dbf90ea8e48a800430cc142"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1bbb122c557a16fafc10354b9d99ebf2f2808a660d78202f10ba9d50786384b9"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:30ca7c3b94708a9d7ae76ff281b2f47d8eaf2579cd05971b5dc681db8caac6e1"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df9270660711670e68803107d55c2b5949c2e0f2e4896da176e1ecfc068b974a"}, - {file = "aiohttp-3.10.10-cp311-cp311-win32.whl", hash = "sha256:aafc8ee9b742ce75044ae9a4d3e60e3d918d15a4c2e08a6c3c3e38fa59b92d94"}, - {file = "aiohttp-3.10.10-cp311-cp311-win_amd64.whl", hash = "sha256:362f641f9071e5f3ee6f8e7d37d5ed0d95aae656adf4ef578313ee585b585959"}, - {file = "aiohttp-3.10.10-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9294bbb581f92770e6ed5c19559e1e99255e4ca604a22c5c6397b2f9dd3ee42c"}, - {file = "aiohttp-3.10.10-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a8fa23fe62c436ccf23ff930149c047f060c7126eae3ccea005f0483f27b2e28"}, - {file = "aiohttp-3.10.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5c6a5b8c7926ba5d8545c7dd22961a107526562da31a7a32fa2456baf040939f"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:007ec22fbc573e5eb2fb7dec4198ef8f6bf2fe4ce20020798b2eb5d0abda6138"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9627cc1a10c8c409b5822a92d57a77f383b554463d1884008e051c32ab1b3742"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50edbcad60d8f0e3eccc68da67f37268b5144ecc34d59f27a02f9611c1d4eec7"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a45d85cf20b5e0d0aa5a8dca27cce8eddef3292bc29d72dcad1641f4ed50aa16"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b00807e2605f16e1e198f33a53ce3c4523114059b0c09c337209ae55e3823a8"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f2d4324a98062be0525d16f768a03e0bbb3b9fe301ceee99611dc9a7953124e6"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:438cd072f75bb6612f2aca29f8bd7cdf6e35e8f160bc312e49fbecab77c99e3a"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:baa42524a82f75303f714108fea528ccacf0386af429b69fff141ffef1c534f9"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a7d8d14fe962153fc681f6366bdec33d4356f98a3e3567782aac1b6e0e40109a"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c1277cd707c465cd09572a774559a3cc7c7a28802eb3a2a9472588f062097205"}, - {file = "aiohttp-3.10.10-cp312-cp312-win32.whl", hash = "sha256:59bb3c54aa420521dc4ce3cc2c3fe2ad82adf7b09403fa1f48ae45c0cbde6628"}, - {file = "aiohttp-3.10.10-cp312-cp312-win_amd64.whl", hash = "sha256:0e1b370d8007c4ae31ee6db7f9a2fe801a42b146cec80a86766e7ad5c4a259cf"}, - {file = "aiohttp-3.10.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ad7593bb24b2ab09e65e8a1d385606f0f47c65b5a2ae6c551db67d6653e78c28"}, - {file = "aiohttp-3.10.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1eb89d3d29adaf533588f209768a9c02e44e4baf832b08118749c5fad191781d"}, - {file = "aiohttp-3.10.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3fe407bf93533a6fa82dece0e74dbcaaf5d684e5a51862887f9eaebe6372cd79"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50aed5155f819873d23520919e16703fc8925e509abbb1a1491b0087d1cd969e"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f05e9727ce409358baa615dbeb9b969db94324a79b5a5cea45d39bdb01d82e6"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dffb610a30d643983aeb185ce134f97f290f8935f0abccdd32c77bed9388b42"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa6658732517ddabe22c9036479eabce6036655ba87a0224c612e1ae6af2087e"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:741a46d58677d8c733175d7e5aa618d277cd9d880301a380fd296975a9cdd7bc"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e00e3505cd80440f6c98c6d69269dcc2a119f86ad0a9fd70bccc59504bebd68a"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ffe595f10566f8276b76dc3a11ae4bb7eba1aac8ddd75811736a15b0d5311414"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bdfcf6443637c148c4e1a20c48c566aa694fa5e288d34b20fcdc58507882fed3"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d183cf9c797a5291e8301790ed6d053480ed94070637bfaad914dd38b0981f67"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:77abf6665ae54000b98b3c742bc6ea1d1fb31c394bcabf8b5d2c1ac3ebfe7f3b"}, - {file = "aiohttp-3.10.10-cp313-cp313-win32.whl", hash = "sha256:4470c73c12cd9109db8277287d11f9dd98f77fc54155fc71a7738a83ffcc8ea8"}, - {file = "aiohttp-3.10.10-cp313-cp313-win_amd64.whl", hash = "sha256:486f7aabfa292719a2753c016cc3a8f8172965cabb3ea2e7f7436c7f5a22a151"}, - {file = "aiohttp-3.10.10-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1b66ccafef7336a1e1f0e389901f60c1d920102315a56df85e49552308fc0486"}, - {file = "aiohttp-3.10.10-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:acd48d5b80ee80f9432a165c0ac8cbf9253eaddb6113269a5e18699b33958dbb"}, - {file = "aiohttp-3.10.10-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3455522392fb15ff549d92fbf4b73b559d5e43dc522588f7eb3e54c3f38beee7"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45c3b868724137f713a38376fef8120c166d1eadd50da1855c112fe97954aed8"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:da1dee8948d2137bb51fbb8a53cce6b1bcc86003c6b42565f008438b806cccd8"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5ce2ce7c997e1971b7184ee37deb6ea9922ef5163c6ee5aa3c274b05f9e12fa"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28529e08fde6f12eba8677f5a8608500ed33c086f974de68cc65ab218713a59d"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7db54c7914cc99d901d93a34704833568d86c20925b2762f9fa779f9cd2e70f"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03a42ac7895406220124c88911ebee31ba8b2d24c98507f4a8bf826b2937c7f2"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7e338c0523d024fad378b376a79faff37fafb3c001872a618cde1d322400a572"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:038f514fe39e235e9fef6717fbf944057bfa24f9b3db9ee551a7ecf584b5b480"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:64f6c17757251e2b8d885d728b6433d9d970573586a78b78ba8929b0f41d045a"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:93429602396f3383a797a2a70e5f1de5df8e35535d7806c9f91df06f297e109b"}, - {file = "aiohttp-3.10.10-cp38-cp38-win32.whl", hash = "sha256:c823bc3971c44ab93e611ab1a46b1eafeae474c0c844aff4b7474287b75fe49c"}, - {file = "aiohttp-3.10.10-cp38-cp38-win_amd64.whl", hash = "sha256:54ca74df1be3c7ca1cf7f4c971c79c2daf48d9aa65dea1a662ae18926f5bc8ce"}, - {file = "aiohttp-3.10.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:01948b1d570f83ee7bbf5a60ea2375a89dfb09fd419170e7f5af029510033d24"}, - {file = "aiohttp-3.10.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9fc1500fd2a952c5c8e3b29aaf7e3cc6e27e9cfc0a8819b3bce48cc1b849e4cc"}, - {file = "aiohttp-3.10.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f614ab0c76397661b90b6851a030004dac502e48260ea10f2441abd2207fbcc7"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00819de9e45d42584bed046314c40ea7e9aea95411b38971082cad449392b08c"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05646ebe6b94cc93407b3bf34b9eb26c20722384d068eb7339de802154d61bc5"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:998f3bd3cfc95e9424a6acd7840cbdd39e45bc09ef87533c006f94ac47296090"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9010c31cd6fa59438da4e58a7f19e4753f7f264300cd152e7f90d4602449762"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ea7ffc6d6d6f8a11e6f40091a1040995cdff02cfc9ba4c2f30a516cb2633554"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ef9c33cc5cbca35808f6c74be11eb7f5f6b14d2311be84a15b594bd3e58b5527"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ce0cdc074d540265bfeb31336e678b4e37316849d13b308607efa527e981f5c2"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:597a079284b7ee65ee102bc3a6ea226a37d2b96d0418cc9047490f231dc09fe8"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:7789050d9e5d0c309c706953e5e8876e38662d57d45f936902e176d19f1c58ab"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e7f8b04d83483577fd9200461b057c9f14ced334dcb053090cea1da9c8321a91"}, - {file = "aiohttp-3.10.10-cp39-cp39-win32.whl", hash = "sha256:c02a30b904282777d872266b87b20ed8cc0d1501855e27f831320f471d54d983"}, - {file = "aiohttp-3.10.10-cp39-cp39-win_amd64.whl", hash = "sha256:edfe3341033a6b53a5c522c802deb2079eee5cbfbb0af032a55064bd65c73a23"}, - {file = "aiohttp-3.10.10.tar.gz", hash = "sha256:0631dd7c9f0822cc61c88586ca76d5b5ada26538097d0f1df510b082bad3411a"}, + {file = "aiohttp-3.11.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a60804bff28662cbcf340a4d61598891f12eea3a66af48ecfdc975ceec21e3c8"}, + {file = "aiohttp-3.11.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4b4fa1cb5f270fb3eab079536b764ad740bb749ce69a94d4ec30ceee1b5940d5"}, + {file = "aiohttp-3.11.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:731468f555656767cda219ab42e033355fe48c85fbe3ba83a349631541715ba2"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb23d8bb86282b342481cad4370ea0853a39e4a32a0042bb52ca6bdde132df43"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f047569d655f81cb70ea5be942ee5d4421b6219c3f05d131f64088c73bb0917f"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd7659baae9ccf94ae5fe8bfaa2c7bc2e94d24611528395ce88d009107e00c6d"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af01e42ad87ae24932138f154105e88da13ce7d202a6de93fafdafb2883a00ef"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5854be2f3e5a729800bac57a8d76af464e160f19676ab6aea74bde18ad19d438"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6526e5fb4e14f4bbf30411216780c9967c20c5a55f2f51d3abd6de68320cc2f3"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:85992ee30a31835fc482468637b3e5bd085fa8fe9392ba0bdcbdc1ef5e9e3c55"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:88a12ad8ccf325a8a5ed80e6d7c3bdc247d66175afedbe104ee2aaca72960d8e"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0a6d3fbf2232e3a08c41eca81ae4f1dff3d8f1a30bae415ebe0af2d2458b8a33"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:84a585799c58b795573c7fa9b84c455adf3e1d72f19a2bf498b54a95ae0d194c"}, + {file = "aiohttp-3.11.11-cp310-cp310-win32.whl", hash = "sha256:bfde76a8f430cf5c5584553adf9926534352251d379dcb266ad2b93c54a29745"}, + {file = "aiohttp-3.11.11-cp310-cp310-win_amd64.whl", hash = "sha256:0fd82b8e9c383af11d2b26f27a478640b6b83d669440c0a71481f7c865a51da9"}, + {file = "aiohttp-3.11.11-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ba74ec819177af1ef7f59063c6d35a214a8fde6f987f7661f4f0eecc468a8f76"}, + {file = "aiohttp-3.11.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4af57160800b7a815f3fe0eba9b46bf28aafc195555f1824555fa2cfab6c1538"}, + {file = "aiohttp-3.11.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ffa336210cf9cd8ed117011085817d00abe4c08f99968deef0013ea283547204"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81b8fe282183e4a3c7a1b72f5ade1094ed1c6345a8f153506d114af5bf8accd9"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af41686ccec6a0f2bdc66686dc0f403c41ac2089f80e2214a0f82d001052c03"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70d1f9dde0e5dd9e292a6d4d00058737052b01f3532f69c0c65818dac26dc287"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:249cc6912405917344192b9f9ea5cd5b139d49e0d2f5c7f70bdfaf6b4dbf3a2e"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0eb98d90b6690827dcc84c246811feeb4e1eea683c0eac6caed7549be9c84665"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ec82bf1fda6cecce7f7b915f9196601a1bd1a3079796b76d16ae4cce6d0ef89b"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9fd46ce0845cfe28f108888b3ab17abff84ff695e01e73657eec3f96d72eef34"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd176afcf8f5d2aed50c3647d4925d0db0579d96f75a31e77cbaf67d8a87742d"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ec2aa89305006fba9ffb98970db6c8221541be7bee4c1d027421d6f6df7d1ce2"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:92cde43018a2e17d48bb09c79e4d4cb0e236de5063ce897a5e40ac7cb4878773"}, + {file = "aiohttp-3.11.11-cp311-cp311-win32.whl", hash = "sha256:aba807f9569455cba566882c8938f1a549f205ee43c27b126e5450dc9f83cc62"}, + {file = "aiohttp-3.11.11-cp311-cp311-win_amd64.whl", hash = "sha256:ae545f31489548c87b0cced5755cfe5a5308d00407000e72c4fa30b19c3220ac"}, + {file = "aiohttp-3.11.11-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e595c591a48bbc295ebf47cb91aebf9bd32f3ff76749ecf282ea7f9f6bb73886"}, + {file = "aiohttp-3.11.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3ea1b59dc06396b0b424740a10a0a63974c725b1c64736ff788a3689d36c02d2"}, + {file = "aiohttp-3.11.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8811f3f098a78ffa16e0ea36dffd577eb031aea797cbdba81be039a4169e242c"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7227b87a355ce1f4bf83bfae4399b1f5bb42e0259cb9405824bd03d2f4336a"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d40f9da8cabbf295d3a9dae1295c69975b86d941bc20f0a087f0477fa0a66231"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ffb3dc385f6bb1568aa974fe65da84723210e5d9707e360e9ecb51f59406cd2e"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8f5f7515f3552d899c61202d99dcb17d6e3b0de777900405611cd747cecd1b8"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3499c7ffbfd9c6a3d8d6a2b01c26639da7e43d47c7b4f788016226b1e711caa8"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8e2bf8029dbf0810c7bfbc3e594b51c4cc9101fbffb583a3923aea184724203c"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b6212a60e5c482ef90f2d788835387070a88d52cf6241d3916733c9176d39eab"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d119fafe7b634dbfa25a8c597718e69a930e4847f0b88e172744be24515140da"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:6fba278063559acc730abf49845d0e9a9e1ba74f85f0ee6efd5803f08b285853"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:92fc484e34b733704ad77210c7957679c5c3877bd1e6b6d74b185e9320cc716e"}, + {file = "aiohttp-3.11.11-cp312-cp312-win32.whl", hash = "sha256:9f5b3c1ed63c8fa937a920b6c1bec78b74ee09593b3f5b979ab2ae5ef60d7600"}, + {file = "aiohttp-3.11.11-cp312-cp312-win_amd64.whl", hash = "sha256:1e69966ea6ef0c14ee53ef7a3d68b564cc408121ea56c0caa2dc918c1b2f553d"}, + {file = "aiohttp-3.11.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:541d823548ab69d13d23730a06f97460f4238ad2e5ed966aaf850d7c369782d9"}, + {file = "aiohttp-3.11.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:929f3ed33743a49ab127c58c3e0a827de0664bfcda566108989a14068f820194"}, + {file = "aiohttp-3.11.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0882c2820fd0132240edbb4a51eb8ceb6eef8181db9ad5291ab3332e0d71df5f"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b63de12e44935d5aca7ed7ed98a255a11e5cb47f83a9fded7a5e41c40277d104"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa54f8ef31d23c506910c21163f22b124facb573bff73930735cf9fe38bf7dff"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a344d5dc18074e3872777b62f5f7d584ae4344cd6006c17ba12103759d407af3"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7fb429ab1aafa1f48578eb315ca45bd46e9c37de11fe45c7f5f4138091e2f1"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c341c7d868750e31961d6d8e60ff040fb9d3d3a46d77fd85e1ab8e76c3e9a5c4"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ed9ee95614a71e87f1a70bc81603f6c6760128b140bc4030abe6abaa988f1c3d"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:de8d38f1c2810fa2a4f1d995a2e9c70bb8737b18da04ac2afbf3971f65781d87"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a9b7371665d4f00deb8f32208c7c5e652059b0fda41cf6dbcac6114a041f1cc2"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:620598717fce1b3bd14dd09947ea53e1ad510317c85dda2c9c65b622edc96b12"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf8d9bfee991d8acc72d060d53860f356e07a50f0e0d09a8dfedea1c554dd0d5"}, + {file = "aiohttp-3.11.11-cp313-cp313-win32.whl", hash = "sha256:9d73ee3725b7a737ad86c2eac5c57a4a97793d9f442599bea5ec67ac9f4bdc3d"}, + {file = "aiohttp-3.11.11-cp313-cp313-win_amd64.whl", hash = "sha256:c7a06301c2fb096bdb0bd25fe2011531c1453b9f2c163c8031600ec73af1cc99"}, + {file = "aiohttp-3.11.11-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3e23419d832d969f659c208557de4a123e30a10d26e1e14b73431d3c13444c2e"}, + {file = "aiohttp-3.11.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:21fef42317cf02e05d3b09c028712e1d73a9606f02467fd803f7c1f39cc59add"}, + {file = "aiohttp-3.11.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1f21bb8d0235fc10c09ce1d11ffbd40fc50d3f08a89e4cf3a0c503dc2562247a"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1642eceeaa5ab6c9b6dfeaaa626ae314d808188ab23ae196a34c9d97efb68350"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2170816e34e10f2fd120f603e951630f8a112e1be3b60963a1f159f5699059a6"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8be8508d110d93061197fd2d6a74f7401f73b6d12f8822bbcd6d74f2b55d71b1"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4eed954b161e6b9b65f6be446ed448ed3921763cc432053ceb606f89d793927e"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6c9af134da4bc9b3bd3e6a70072509f295d10ee60c697826225b60b9959acdd"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:44167fc6a763d534a6908bdb2592269b4bf30a03239bcb1654781adf5e49caf1"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:479b8c6ebd12aedfe64563b85920525d05d394b85f166b7873c8bde6da612f9c"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:10b4ff0ad793d98605958089fabfa350e8e62bd5d40aa65cdc69d6785859f94e"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b540bd67cfb54e6f0865ceccd9979687210d7ed1a1cc8c01f8e67e2f1e883d28"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1dac54e8ce2ed83b1f6b1a54005c87dfed139cf3f777fdc8afc76e7841101226"}, + {file = "aiohttp-3.11.11-cp39-cp39-win32.whl", hash = "sha256:568c1236b2fde93b7720f95a890741854c1200fba4a3471ff48b2934d2d93fd3"}, + {file = "aiohttp-3.11.11-cp39-cp39-win_amd64.whl", hash = "sha256:943a8b052e54dfd6439fd7989f67fc6a7f2138d0a2cf0a7de5f18aa4fe7eb3b1"}, + {file = "aiohttp-3.11.11.tar.gz", hash = "sha256:bb49c7f1e6ebf3821a42d81d494f538107610c3a705987f53068546b0e90303e"}, ] [package.dependencies] aiohappyeyeballs = ">=2.3.0" aiosignal = ">=1.1.2" -async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} +async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" -yarl = ">=1.12.0,<2.0" +propcache = ">=0.2.0" +yarl = ">=1.17.0,<2.0" [package.extras] speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] [[package]] name = "aiosignal" -version = "1.3.1" +version = "1.3.2" description = "aiosignal: a list of registered asynchronous callbacks" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, + {file = "aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5"}, + {file = "aiosignal-1.3.2.tar.gz", hash = "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54"}, ] [package.dependencies] @@ -148,37 +134,61 @@ files = [ {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, ] +[[package]] +name = "anthropic" +version = "0.42.0" +description = "The official Python library for the anthropic API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anthropic-0.42.0-py3-none-any.whl", hash = "sha256:46775f65b723c078a2ac9e9de44a46db5c6a4fabeacfd165e5ea78e6817f4eff"}, + {file = "anthropic-0.42.0.tar.gz", hash = "sha256:bf8b0ed8c8cb2c2118038f29c58099d2f99f7847296cafdaa853910bfff4edf4"}, +] + +[package.dependencies] +anyio = ">=3.5.0,<5" +distro = ">=1.7.0,<2" +httpx = ">=0.23.0,<1" +jiter = ">=0.4.0,<1" +pydantic = ">=1.9.0,<3" +sniffio = "*" +typing-extensions = ">=4.10,<5" + +[package.extras] +bedrock = ["boto3 (>=1.28.57)", "botocore (>=1.31.57)"] +vertex = ["google-auth (>=2,<3)"] + [[package]] name = "anyio" -version = "4.6.2.post1" +version = "4.8.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.9" files = [ - {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, - {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, + {file = "anyio-4.8.0-py3-none-any.whl", hash = "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a"}, + {file = "anyio-4.8.0.tar.gz", hash = "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a"}, ] [package.dependencies] exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" -typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] -doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] trio = ["trio (>=0.26.1)"] [[package]] name = "astroid" -version = "3.3.5" +version = "3.3.8" description = "An abstract syntax tree for Python with inference support." optional = false python-versions = ">=3.9.0" files = [ - {file = "astroid-3.3.5-py3-none-any.whl", hash = "sha256:a9d1c946ada25098d790e079ba2a1b112157278f3fb7e718ae6a9252f5835dc8"}, - {file = "astroid-3.3.5.tar.gz", hash = "sha256:5cfc40ae9f68311075d27ef68a4841bdc5cc7f6cf86671b49f00607d30188e2d"}, + {file = "astroid-3.3.8-py3-none-any.whl", hash = "sha256:187ccc0c248bfbba564826c26f070494f7bc964fd286b6d9fff4420e55de828c"}, + {file = "astroid-3.3.8.tar.gz", hash = "sha256:a88c7994f914a4ea8572fac479459f4955eeccc877be3f2d959a33273b0cf40b"}, ] [package.dependencies] @@ -186,30 +196,30 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} [[package]] name = "async-timeout" -version = "4.0.3" +version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, - {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, ] [[package]] name = "attrs" -version = "24.2.0" +version = "24.3.0" description = "Classes Without Boilerplate" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, - {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, + {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, + {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, ] [package.extras] benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] @@ -327,138 +337,125 @@ beautifulsoup4 = "*" [[package]] name = "certifi" -version = "2024.8.30" +version = "2024.12.14" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, + {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, ] [[package]] name = "charset-normalizer" -version = "3.4.0" +version = "3.4.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false -python-versions = ">=3.7.0" -files = [ - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, - {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, - {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, +python-versions = ">=3.7" +files = [ + {file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win32.whl", hash = "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win32.whl", hash = "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win32.whl", hash = "sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765"}, + {file = "charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85"}, + {file = "charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3"}, ] [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, ] [package.dependencies] @@ -650,13 +647,13 @@ files = [ [[package]] name = "fsspec" -version = "2024.10.0" +version = "2024.12.0" description = "File-system specification" optional = false python-versions = ">=3.8" files = [ - {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, - {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, + {file = "fsspec-2024.12.0-py3-none-any.whl", hash = "sha256:b520aed47ad9804237ff878b504267a3b0b441e97508bd6d2d8774e3db85cee2"}, + {file = "fsspec-2024.12.0.tar.gz", hash = "sha256:670700c977ed2fb51e0d9f9253177ed20cbde4a3e5c0283cc5385b5870c8533f"}, ] [package.extras] @@ -786,13 +783,13 @@ files = [ [[package]] name = "httpcore" -version = "1.0.6" +version = "1.0.7" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"}, - {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"}, + {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, + {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, ] [package.dependencies] @@ -832,13 +829,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "0.26.2" +version = "0.27.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.26.2-py3-none-any.whl", hash = "sha256:98c2a5a8e786c7b2cb6fdeb2740893cba4d53e312572ed3d8afafda65b128c46"}, - {file = "huggingface_hub-0.26.2.tar.gz", hash = "sha256:b100d853465d965733964d123939ba287da60a547087783ddff8a323f340332b"}, + {file = "huggingface_hub-0.27.1-py3-none-any.whl", hash = "sha256:1c5155ca7d60b60c2e2fc38cbb3ffb7f7c3adf48f824015b219af9061771daec"}, + {file = "huggingface_hub-0.27.1.tar.gz", hash = "sha256:c004463ca870283909d715d20f066ebd6968c2207dae9393fdffb3c1d4d8f98b"}, ] [package.dependencies] @@ -928,13 +925,13 @@ colors = ["colorama (>=0.4.6)"] [[package]] name = "jinja2" -version = "3.1.4" +version = "3.1.5" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, + {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, + {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, ] [package.dependencies] @@ -945,95 +942,98 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jiter" -version = "0.7.1" +version = "0.8.2" description = "Fast iterable JSON parser." optional = false python-versions = ">=3.8" files = [ - {file = "jiter-0.7.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:262e96d06696b673fad6f257e6a0abb6e873dc22818ca0e0600f4a1189eb334f"}, - {file = "jiter-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be6de02939aac5be97eb437f45cfd279b1dc9de358b13ea6e040e63a3221c40d"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935f10b802bc1ce2b2f61843e498c7720aa7f4e4bb7797aa8121eab017293c3d"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9cd3cccccabf5064e4bb3099c87bf67db94f805c1e62d1aefd2b7476e90e0ee2"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4aa919ebfc5f7b027cc368fe3964c0015e1963b92e1db382419dadb098a05192"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ae2d01e82c94491ce4d6f461a837f63b6c4e6dd5bb082553a70c509034ff3d4"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f9568cd66dbbdab67ae1b4c99f3f7da1228c5682d65913e3f5f95586b3cb9a9"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ecbf4e20ec2c26512736284dc1a3f8ed79b6ca7188e3b99032757ad48db97dc"}, - {file = "jiter-0.7.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b1a0508fddc70ce00b872e463b387d49308ef02b0787992ca471c8d4ba1c0fa1"}, - {file = "jiter-0.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f84c9996664c460f24213ff1e5881530abd8fafd82058d39af3682d5fd2d6316"}, - {file = "jiter-0.7.1-cp310-none-win32.whl", hash = "sha256:c915e1a1960976ba4dfe06551ea87063b2d5b4d30759012210099e712a414d9f"}, - {file = "jiter-0.7.1-cp310-none-win_amd64.whl", hash = "sha256:75bf3b7fdc5c0faa6ffffcf8028a1f974d126bac86d96490d1b51b3210aa0f3f"}, - {file = "jiter-0.7.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ad04a23a91f3d10d69d6c87a5f4471b61c2c5cd6e112e85136594a02043f462c"}, - {file = "jiter-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e47a554de88dff701226bb5722b7f1b6bccd0b98f1748459b7e56acac2707a5"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e44fff69c814a2e96a20b4ecee3e2365e9b15cf5fe4e00869d18396daa91dab"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:df0a1d05081541b45743c965436f8b5a1048d6fd726e4a030113a2699a6046ea"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f22cf8f236a645cb6d8ffe2a64edb5d2b66fb148bf7c75eea0cb36d17014a7bc"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da8589f50b728ea4bf22e0632eefa125c8aa9c38ed202a5ee6ca371f05eeb3ff"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f20de711224f2ca2dbb166a8d512f6ff48c9c38cc06b51f796520eb4722cc2ce"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8a9803396032117b85ec8cbf008a54590644a062fedd0425cbdb95e4b2b60479"}, - {file = "jiter-0.7.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3d8bae77c82741032e9d89a4026479061aba6e646de3bf5f2fc1ae2bbd9d06e0"}, - {file = "jiter-0.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3dc9939e576bbc68c813fc82f6620353ed68c194c7bcf3d58dc822591ec12490"}, - {file = "jiter-0.7.1-cp311-none-win32.whl", hash = "sha256:f7605d24cd6fab156ec89e7924578e21604feee9c4f1e9da34d8b67f63e54892"}, - {file = "jiter-0.7.1-cp311-none-win_amd64.whl", hash = "sha256:f3ea649e7751a1a29ea5ecc03c4ada0a833846c59c6da75d747899f9b48b7282"}, - {file = "jiter-0.7.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ad36a1155cbd92e7a084a568f7dc6023497df781adf2390c345dd77a120905ca"}, - {file = "jiter-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7ba52e6aaed2dc5c81a3d9b5e4ab95b039c4592c66ac973879ba57c3506492bb"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b7de0b6f6728b678540c7927587e23f715284596724be203af952418acb8a2d"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9463b62bd53c2fb85529c700c6a3beb2ee54fde8bef714b150601616dcb184a6"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:627164ec01d28af56e1f549da84caf0fe06da3880ebc7b7ee1ca15df106ae172"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25d0e5bf64e368b0aa9e0a559c3ab2f9b67e35fe7269e8a0d81f48bbd10e8963"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c244261306f08f8008b3087059601997016549cb8bb23cf4317a4827f07b7d74"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7ded4e4b75b68b843b7cea5cd7c55f738c20e1394c68c2cb10adb655526c5f1b"}, - {file = "jiter-0.7.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:80dae4f1889b9d09e5f4de6b58c490d9c8ce7730e35e0b8643ab62b1538f095c"}, - {file = "jiter-0.7.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5970cf8ec943b51bce7f4b98d2e1ed3ada170c2a789e2db3cb484486591a176a"}, - {file = "jiter-0.7.1-cp312-none-win32.whl", hash = "sha256:701d90220d6ecb3125d46853c8ca8a5bc158de8c49af60fd706475a49fee157e"}, - {file = "jiter-0.7.1-cp312-none-win_amd64.whl", hash = "sha256:7824c3ecf9ecf3321c37f4e4d4411aad49c666ee5bc2a937071bdd80917e4533"}, - {file = "jiter-0.7.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:097676a37778ba3c80cb53f34abd6943ceb0848263c21bf423ae98b090f6c6ba"}, - {file = "jiter-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3298af506d4271257c0a8f48668b0f47048d69351675dd8500f22420d4eec378"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12fd88cfe6067e2199964839c19bd2b422ca3fd792949b8f44bb8a4e7d21946a"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dacca921efcd21939123c8ea8883a54b9fa7f6545c8019ffcf4f762985b6d0c8"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de3674a5fe1f6713a746d25ad9c32cd32fadc824e64b9d6159b3b34fd9134143"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65df9dbae6d67e0788a05b4bad5706ad40f6f911e0137eb416b9eead6ba6f044"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ba9a358d59a0a55cccaa4957e6ae10b1a25ffdabda863c0343c51817610501d"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:576eb0f0c6207e9ede2b11ec01d9c2182973986514f9c60bc3b3b5d5798c8f50"}, - {file = "jiter-0.7.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:e550e29cdf3577d2c970a18f3959e6b8646fd60ef1b0507e5947dc73703b5627"}, - {file = "jiter-0.7.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:81d968dbf3ce0db2e0e4dec6b0a0d5d94f846ee84caf779b07cab49f5325ae43"}, - {file = "jiter-0.7.1-cp313-none-win32.whl", hash = "sha256:f892e547e6e79a1506eb571a676cf2f480a4533675f834e9ae98de84f9b941ac"}, - {file = "jiter-0.7.1-cp313-none-win_amd64.whl", hash = "sha256:0302f0940b1455b2a7fb0409b8d5b31183db70d2b07fd177906d83bf941385d1"}, - {file = "jiter-0.7.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:c65a3ce72b679958b79d556473f192a4dfc5895e8cc1030c9f4e434690906076"}, - {file = "jiter-0.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e80052d3db39f9bb8eb86d207a1be3d9ecee5e05fdec31380817f9609ad38e60"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70a497859c4f3f7acd71c8bd89a6f9cf753ebacacf5e3e799138b8e1843084e3"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c1288bc22b9e36854a0536ba83666c3b1fb066b811019d7b682c9cf0269cdf9f"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b096ca72dd38ef35675e1d3b01785874315182243ef7aea9752cb62266ad516f"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8dbbd52c50b605af13dbee1a08373c520e6fcc6b5d32f17738875847fea4e2cd"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af29c5c6eb2517e71ffa15c7ae9509fa5e833ec2a99319ac88cc271eca865519"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f114a4df1e40c03c0efbf974b376ed57756a1141eb27d04baee0680c5af3d424"}, - {file = "jiter-0.7.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:191fbaee7cf46a9dd9b817547bf556facde50f83199d07fc48ebeff4082f9df4"}, - {file = "jiter-0.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e2b445e5ee627fb4ee6bbceeb486251e60a0c881a8e12398dfdff47c56f0723"}, - {file = "jiter-0.7.1-cp38-none-win32.whl", hash = "sha256:47ac4c3cf8135c83e64755b7276339b26cd3c7ddadf9e67306ace4832b283edf"}, - {file = "jiter-0.7.1-cp38-none-win_amd64.whl", hash = "sha256:60b49c245cd90cde4794f5c30f123ee06ccf42fb8730a019a2870cd005653ebd"}, - {file = "jiter-0.7.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8f212eeacc7203256f526f550d105d8efa24605828382cd7d296b703181ff11d"}, - {file = "jiter-0.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d9e247079d88c00e75e297e6cb3a18a039ebcd79fefc43be9ba4eb7fb43eb726"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0aacaa56360139c53dcf352992b0331f4057a0373bbffd43f64ba0c32d2d155"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bc1b55314ca97dbb6c48d9144323896e9c1a25d41c65bcb9550b3e0c270ca560"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f281aae41b47e90deb70e7386558e877a8e62e1693e0086f37d015fa1c102289"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:93c20d2730a84d43f7c0b6fb2579dc54335db742a59cf9776d0b80e99d587382"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e81ccccd8069110e150613496deafa10da2f6ff322a707cbec2b0d52a87b9671"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0a7d5e85766eff4c9be481d77e2226b4c259999cb6862ccac5ef6621d3c8dcce"}, - {file = "jiter-0.7.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f52ce5799df5b6975439ecb16b1e879d7655e1685b6e3758c9b1b97696313bfb"}, - {file = "jiter-0.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e0c91a0304373fdf97d56f88356a010bba442e6d995eb7773cbe32885b71cdd8"}, - {file = "jiter-0.7.1-cp39-none-win32.whl", hash = "sha256:5c08adf93e41ce2755970e8aa95262298afe2bf58897fb9653c47cd93c3c6cdc"}, - {file = "jiter-0.7.1-cp39-none-win_amd64.whl", hash = "sha256:6592f4067c74176e5f369228fb2995ed01400c9e8e1225fb73417183a5e635f0"}, - {file = "jiter-0.7.1.tar.gz", hash = "sha256:448cf4f74f7363c34cdef26214da527e8eeffd88ba06d0b80b485ad0667baf5d"}, + {file = "jiter-0.8.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ca8577f6a413abe29b079bc30f907894d7eb07a865c4df69475e868d73e71c7b"}, + {file = "jiter-0.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b25bd626bde7fb51534190c7e3cb97cee89ee76b76d7585580e22f34f5e3f393"}, + {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5c826a221851a8dc028eb6d7d6429ba03184fa3c7e83ae01cd6d3bd1d4bd17d"}, + {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d35c864c2dff13dfd79fb070fc4fc6235d7b9b359efe340e1261deb21b9fcb66"}, + {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f557c55bc2b7676e74d39d19bcb8775ca295c7a028246175d6a8b431e70835e5"}, + {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:580ccf358539153db147e40751a0b41688a5ceb275e6f3e93d91c9467f42b2e3"}, + {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af102d3372e917cffce49b521e4c32c497515119dc7bd8a75665e90a718bbf08"}, + {file = "jiter-0.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cadcc978f82397d515bb2683fc0d50103acff2a180552654bb92d6045dec2c49"}, + {file = "jiter-0.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ba5bdf56969cad2019d4e8ffd3f879b5fdc792624129741d3d83fc832fef8c7d"}, + {file = "jiter-0.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3b94a33a241bee9e34b8481cdcaa3d5c2116f575e0226e421bed3f7a6ea71cff"}, + {file = "jiter-0.8.2-cp310-cp310-win32.whl", hash = "sha256:6e5337bf454abddd91bd048ce0dca5134056fc99ca0205258766db35d0a2ea43"}, + {file = "jiter-0.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:4a9220497ca0cb1fe94e3f334f65b9b5102a0b8147646118f020d8ce1de70105"}, + {file = "jiter-0.8.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2dd61c5afc88a4fda7d8b2cf03ae5947c6ac7516d32b7a15bf4b49569a5c076b"}, + {file = "jiter-0.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a6c710d657c8d1d2adbbb5c0b0c6bfcec28fd35bd6b5f016395f9ac43e878a15"}, + {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9584de0cd306072635fe4b89742bf26feae858a0683b399ad0c2509011b9dc0"}, + {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5a90a923338531b7970abb063cfc087eebae6ef8ec8139762007188f6bc69a9f"}, + {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21974d246ed0181558087cd9f76e84e8321091ebfb3a93d4c341479a736f099"}, + {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32475a42b2ea7b344069dc1e81445cfc00b9d0e3ca837f0523072432332e9f74"}, + {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b9931fd36ee513c26b5bf08c940b0ac875de175341cbdd4fa3be109f0492586"}, + {file = "jiter-0.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce0820f4a3a59ddced7fce696d86a096d5cc48d32a4183483a17671a61edfddc"}, + {file = "jiter-0.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8ffc86ae5e3e6a93765d49d1ab47b6075a9c978a2b3b80f0f32628f39caa0c88"}, + {file = "jiter-0.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5127dc1abd809431172bc3fbe8168d6b90556a30bb10acd5ded41c3cfd6f43b6"}, + {file = "jiter-0.8.2-cp311-cp311-win32.whl", hash = "sha256:66227a2c7b575720c1871c8800d3a0122bb8ee94edb43a5685aa9aceb2782d44"}, + {file = "jiter-0.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:cde031d8413842a1e7501e9129b8e676e62a657f8ec8166e18a70d94d4682855"}, + {file = "jiter-0.8.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e6ec2be506e7d6f9527dae9ff4b7f54e68ea44a0ef6b098256ddf895218a2f8f"}, + {file = "jiter-0.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76e324da7b5da060287c54f2fabd3db5f76468006c811831f051942bf68c9d44"}, + {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:180a8aea058f7535d1c84183c0362c710f4750bef66630c05f40c93c2b152a0f"}, + {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:025337859077b41548bdcbabe38698bcd93cfe10b06ff66617a48ff92c9aec60"}, + {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ecff0dc14f409599bbcafa7e470c00b80f17abc14d1405d38ab02e4b42e55b57"}, + {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ffd9fee7d0775ebaba131f7ca2e2d83839a62ad65e8e02fe2bd8fc975cedeb9e"}, + {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14601dcac4889e0a1c75ccf6a0e4baf70dbc75041e51bcf8d0e9274519df6887"}, + {file = "jiter-0.8.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:92249669925bc1c54fcd2ec73f70f2c1d6a817928480ee1c65af5f6b81cdf12d"}, + {file = "jiter-0.8.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e725edd0929fa79f8349ab4ec7f81c714df51dc4e991539a578e5018fa4a7152"}, + {file = "jiter-0.8.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bf55846c7b7a680eebaf9c3c48d630e1bf51bdf76c68a5f654b8524335b0ad29"}, + {file = "jiter-0.8.2-cp312-cp312-win32.whl", hash = "sha256:7efe4853ecd3d6110301665a5178b9856be7e2a9485f49d91aa4d737ad2ae49e"}, + {file = "jiter-0.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:83c0efd80b29695058d0fd2fa8a556490dbce9804eac3e281f373bbc99045f6c"}, + {file = "jiter-0.8.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:ca1f08b8e43dc3bd0594c992fb1fd2f7ce87f7bf0d44358198d6da8034afdf84"}, + {file = "jiter-0.8.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5672a86d55416ccd214c778efccf3266b84f87b89063b582167d803246354be4"}, + {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58dc9bc9767a1101f4e5e22db1b652161a225874d66f0e5cb8e2c7d1c438b587"}, + {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:37b2998606d6dadbb5ccda959a33d6a5e853252d921fec1792fc902351bb4e2c"}, + {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ab9a87f3784eb0e098f84a32670cfe4a79cb6512fd8f42ae3d0709f06405d18"}, + {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:79aec8172b9e3c6d05fd4b219d5de1ac616bd8da934107325a6c0d0e866a21b6"}, + {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:711e408732d4e9a0208008e5892c2966b485c783cd2d9a681f3eb147cf36c7ef"}, + {file = "jiter-0.8.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:653cf462db4e8c41995e33d865965e79641ef45369d8a11f54cd30888b7e6ff1"}, + {file = "jiter-0.8.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:9c63eaef32b7bebac8ebebf4dabebdbc6769a09c127294db6babee38e9f405b9"}, + {file = "jiter-0.8.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:eb21aaa9a200d0a80dacc7a81038d2e476ffe473ffdd9c91eb745d623561de05"}, + {file = "jiter-0.8.2-cp313-cp313-win32.whl", hash = "sha256:789361ed945d8d42850f919342a8665d2dc79e7e44ca1c97cc786966a21f627a"}, + {file = "jiter-0.8.2-cp313-cp313-win_amd64.whl", hash = "sha256:ab7f43235d71e03b941c1630f4b6e3055d46b6cb8728a17663eaac9d8e83a865"}, + {file = "jiter-0.8.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b426f72cd77da3fec300ed3bc990895e2dd6b49e3bfe6c438592a3ba660e41ca"}, + {file = "jiter-0.8.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2dd880785088ff2ad21ffee205e58a8c1ddabc63612444ae41e5e4b321b39c0"}, + {file = "jiter-0.8.2-cp313-cp313t-win_amd64.whl", hash = "sha256:3ac9f578c46f22405ff7f8b1f5848fb753cc4b8377fbec8470a7dc3997ca7566"}, + {file = "jiter-0.8.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:9e1fa156ee9454642adb7e7234a383884452532bc9d53d5af2d18d98ada1d79c"}, + {file = "jiter-0.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cf5dfa9956d96ff2efb0f8e9c7d055904012c952539a774305aaaf3abdf3d6c"}, + {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e52bf98c7e727dd44f7c4acb980cb988448faeafed8433c867888268899b298b"}, + {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a2ecaa3c23e7a7cf86d00eda3390c232f4d533cd9ddea4b04f5d0644faf642c5"}, + {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08d4c92bf480e19fc3f2717c9ce2aa31dceaa9163839a311424b6862252c943e"}, + {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d9a1eded738299ba8e106c6779ce5c3893cffa0e32e4485d680588adae6db8"}, + {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d20be8b7f606df096e08b0b1b4a3c6f0515e8dac296881fe7461dfa0fb5ec817"}, + {file = "jiter-0.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d33f94615fcaf872f7fd8cd98ac3b429e435c77619777e8a449d9d27e01134d1"}, + {file = "jiter-0.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:317b25e98a35ffec5c67efe56a4e9970852632c810d35b34ecdd70cc0e47b3b6"}, + {file = "jiter-0.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fc9043259ee430ecd71d178fccabd8c332a3bf1e81e50cae43cc2b28d19e4cb7"}, + {file = "jiter-0.8.2-cp38-cp38-win32.whl", hash = "sha256:fc5adda618205bd4678b146612ce44c3cbfdee9697951f2c0ffdef1f26d72b63"}, + {file = "jiter-0.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:cd646c827b4f85ef4a78e4e58f4f5854fae0caf3db91b59f0d73731448a970c6"}, + {file = "jiter-0.8.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e41e75344acef3fc59ba4765df29f107f309ca9e8eace5baacabd9217e52a5ee"}, + {file = "jiter-0.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f22b16b35d5c1df9dfd58843ab2cd25e6bf15191f5a236bed177afade507bfc"}, + {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7200b8f7619d36aa51c803fd52020a2dfbea36ffec1b5e22cab11fd34d95a6d"}, + {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:70bf4c43652cc294040dbb62256c83c8718370c8b93dd93d934b9a7bf6c4f53c"}, + {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f9d471356dc16f84ed48768b8ee79f29514295c7295cb41e1133ec0b2b8d637d"}, + {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:859e8eb3507894093d01929e12e267f83b1d5f6221099d3ec976f0c995cb6bd9"}, + {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaa58399c01db555346647a907b4ef6d4f584b123943be6ed5588c3f2359c9f4"}, + {file = "jiter-0.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8f2d5ed877f089862f4c7aacf3a542627c1496f972a34d0474ce85ee7d939c27"}, + {file = "jiter-0.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:03c9df035d4f8d647f8c210ddc2ae0728387275340668fb30d2421e17d9a0841"}, + {file = "jiter-0.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8bd2a824d08d8977bb2794ea2682f898ad3d8837932e3a74937e93d62ecbb637"}, + {file = "jiter-0.8.2-cp39-cp39-win32.whl", hash = "sha256:ca29b6371ebc40e496995c94b988a101b9fbbed48a51190a4461fcb0a68b4a36"}, + {file = "jiter-0.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:1c0dfbd1be3cbefc7510102370d86e35d1d53e5a93d48519688b1bf0f761160a"}, + {file = "jiter-0.8.2.tar.gz", hash = "sha256:cd73d3e740666d0e639f678adb176fad25c1bcbdae88d8d7b857e1783bb4212d"}, ] [[package]] name = "json-repair" -version = "0.30.1" +version = "0.30.3" description = "A package to repair broken json strings" optional = false python-versions = ">=3.9" files = [ - {file = "json_repair-0.30.1-py3-none-any.whl", hash = "sha256:6fa8a05d246e282df2f812fa542bd837d671d7774eaae11191aabaac97d41e33"}, - {file = "json_repair-0.30.1.tar.gz", hash = "sha256:5f075c4e3b098d78fb6cd60c34aec07a4517f14e9d423ad5364214b0e870e218"}, + {file = "json_repair-0.30.3-py3-none-any.whl", hash = "sha256:63bb588162b0958ae93d85356ecbe54c06b8c33f8a4834f93fa2719ea669804e"}, + {file = "json_repair-0.30.3.tar.gz", hash = "sha256:0ac56e7ae9253ee9c507a7e1a3a26799c9b0bbe5e2bec1b2cc5053e90d5b05e3"}, ] [[package]] @@ -1073,41 +1073,41 @@ referencing = ">=0.31.0" [[package]] name = "litellm" -version = "1.52.9" +version = "1.57.0" description = "Library to easily interface with LLM API providers" optional = false python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" files = [ - {file = "litellm-1.52.9-py3-none-any.whl", hash = "sha256:a1ef5561d220d77059a359da497f0ab04c721205c6795f151b07be5bbe51fe45"}, - {file = "litellm-1.52.9.tar.gz", hash = "sha256:73a05fed76cfac4357ee4117f28608209db891223fb9c6e03dddfe1723666437"}, + {file = "litellm-1.57.0-py3-none-any.whl", hash = "sha256:339aec6f3ecac2035bf6311aa8913ce587c9aca2dc7d72a63a210c659e9721ca"}, + {file = "litellm-1.57.0.tar.gz", hash = "sha256:53a6f2bd9575823e102f7d18dde5cbd2d48eed027cecbb585f18a208605b34c5"}, ] [package.dependencies] aiohttp = "*" click = "*" +httpx = ">=0.23.0,<0.28.0" importlib-metadata = ">=6.8.0" jinja2 = ">=3.1.2,<4.0.0" jsonschema = ">=4.22.0,<5.0.0" -openai = ">=1.54.0" +openai = ">=1.55.3" pydantic = ">=2.0.0,<3.0.0" python-dotenv = ">=0.2.0" -requests = ">=2.31.0,<3.0.0" tiktoken = ">=0.7.0" tokenizers = "*" [package.extras] extra-proxy = ["azure-identity (>=1.15.0,<2.0.0)", "azure-keyvault-secrets (>=4.8.0,<5.0.0)", "google-cloud-kms (>=2.21.3,<3.0.0)", "prisma (==0.11.0)", "resend (>=0.8.0,<0.9.0)"] -proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "cryptography (>=42.0.5,<43.0.0)", "fastapi (>=0.111.0,<0.112.0)", "fastapi-sso (>=0.10.0,<0.11.0)", "gunicorn (>=22.0.0,<23.0.0)", "orjson (>=3.9.7,<4.0.0)", "pynacl (>=1.5.0,<2.0.0)", "python-multipart (>=0.0.9,<0.0.10)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"] +proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "cryptography (>=43.0.1,<44.0.0)", "fastapi (>=0.115.5,<0.116.0)", "fastapi-sso (>=0.16.0,<0.17.0)", "gunicorn (>=22.0.0,<23.0.0)", "orjson (>=3.9.7,<4.0.0)", "pynacl (>=1.5.0,<2.0.0)", "python-multipart (>=0.0.18,<0.0.19)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"] [[package]] name = "loguru" -version = "0.7.2" +version = "0.7.3" description = "Python logging made (stupidly) simple" optional = false -python-versions = ">=3.5" +python-versions = "<4.0,>=3.5" files = [ - {file = "loguru-0.7.2-py3-none-any.whl", hash = "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb"}, - {file = "loguru-0.7.2.tar.gz", hash = "sha256:e671a53522515f34fd406340ee968cb9ecafbc4b36c679da03c18fd8d0bd51ac"}, + {file = "loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c"}, + {file = "loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6"}, ] [package.dependencies] @@ -1115,7 +1115,7 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"] +dev = ["Sphinx (==8.1.3)", "build (==1.2.2)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.5.0)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.13.0)", "mypy (==v1.4.1)", "myst-parser (==4.0.0)", "pre-commit (==4.0.1)", "pytest (==6.1.2)", "pytest (==8.3.2)", "pytest-cov (==2.12.1)", "pytest-cov (==5.0.0)", "pytest-cov (==6.0.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.1.0)", "sphinx-rtd-theme (==3.0.2)", "tox (==3.27.1)", "tox (==4.23.2)", "twine (==6.0.1)"] [[package]] name = "lxml" @@ -1484,13 +1484,13 @@ files = [ [[package]] name = "openai" -version = "1.54.4" +version = "1.59.3" description = "The official Python library for the openai API" optional = false python-versions = ">=3.8" files = [ - {file = "openai-1.54.4-py3-none-any.whl", hash = "sha256:0d95cef99346bf9b6d7fbf57faf61a673924c3e34fa8af84c9ffe04660673a7e"}, - {file = "openai-1.54.4.tar.gz", hash = "sha256:50f3656e45401c54e973fa05dc29f3f0b0d19348d685b2f7ddb4d92bf7b1b6bf"}, + {file = "openai-1.59.3-py3-none-any.whl", hash = "sha256:b041887a0d8f3e70d1fc6ffbb2bf7661c3b9a2f3e806c04bf42f572b9ac7bc37"}, + {file = "openai-1.59.3.tar.gz", hash = "sha256:7f7fff9d8729968588edf1524e73266e8593bb6cab09298340efb755755bb66f"}, ] [package.dependencies] @@ -1505,16 +1505,17 @@ typing-extensions = ">=4.11,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +realtime = ["websockets (>=13,<15)"] [[package]] name = "packaging" -version = "24.1" +version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] [[package]] @@ -1541,93 +1542,89 @@ files = [ [[package]] name = "pillow" -version = "11.0.0" +version = "11.1.0" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.9" files = [ - {file = "pillow-11.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6619654954dc4936fcff82db8eb6401d3159ec6be81e33c6000dfd76ae189947"}, - {file = "pillow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b3c5ac4bed7519088103d9450a1107f76308ecf91d6dabc8a33a2fcfb18d0fba"}, - {file = "pillow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a65149d8ada1055029fcb665452b2814fe7d7082fcb0c5bed6db851cb69b2086"}, - {file = "pillow-11.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88a58d8ac0cc0e7f3a014509f0455248a76629ca9b604eca7dc5927cc593c5e9"}, - {file = "pillow-11.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c26845094b1af3c91852745ae78e3ea47abf3dbcd1cf962f16b9a5fbe3ee8488"}, - {file = "pillow-11.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1a61b54f87ab5786b8479f81c4b11f4d61702830354520837f8cc791ebba0f5f"}, - {file = "pillow-11.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:674629ff60030d144b7bca2b8330225a9b11c482ed408813924619c6f302fdbb"}, - {file = "pillow-11.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:598b4e238f13276e0008299bd2482003f48158e2b11826862b1eb2ad7c768b97"}, - {file = "pillow-11.0.0-cp310-cp310-win32.whl", hash = "sha256:9a0f748eaa434a41fccf8e1ee7a3eed68af1b690e75328fd7a60af123c193b50"}, - {file = "pillow-11.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a5629742881bcbc1f42e840af185fd4d83a5edeb96475a575f4da50d6ede337c"}, - {file = "pillow-11.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:ee217c198f2e41f184f3869f3e485557296d505b5195c513b2bfe0062dc537f1"}, - {file = "pillow-11.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1c1d72714f429a521d8d2d018badc42414c3077eb187a59579f28e4270b4b0fc"}, - {file = "pillow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:499c3a1b0d6fc8213519e193796eb1a86a1be4b1877d678b30f83fd979811d1a"}, - {file = "pillow-11.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8b2351c85d855293a299038e1f89db92a2f35e8d2f783489c6f0b2b5f3fe8a3"}, - {file = "pillow-11.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f4dba50cfa56f910241eb7f883c20f1e7b1d8f7d91c750cd0b318bad443f4d5"}, - {file = "pillow-11.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5ddbfd761ee00c12ee1be86c9c0683ecf5bb14c9772ddbd782085779a63dd55b"}, - {file = "pillow-11.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:45c566eb10b8967d71bf1ab8e4a525e5a93519e29ea071459ce517f6b903d7fa"}, - {file = "pillow-11.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b4fd7bd29610a83a8c9b564d457cf5bd92b4e11e79a4ee4716a63c959699b306"}, - {file = "pillow-11.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cb929ca942d0ec4fac404cbf520ee6cac37bf35be479b970c4ffadf2b6a1cad9"}, - {file = "pillow-11.0.0-cp311-cp311-win32.whl", hash = "sha256:006bcdd307cc47ba43e924099a038cbf9591062e6c50e570819743f5607404f5"}, - {file = "pillow-11.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:52a2d8323a465f84faaba5236567d212c3668f2ab53e1c74c15583cf507a0291"}, - {file = "pillow-11.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:16095692a253047fe3ec028e951fa4221a1f3ed3d80c397e83541a3037ff67c9"}, - {file = "pillow-11.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d2c0a187a92a1cb5ef2c8ed5412dd8d4334272617f532d4ad4de31e0495bd923"}, - {file = "pillow-11.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:084a07ef0821cfe4858fe86652fffac8e187b6ae677e9906e192aafcc1b69903"}, - {file = "pillow-11.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8069c5179902dcdce0be9bfc8235347fdbac249d23bd90514b7a47a72d9fecf4"}, - {file = "pillow-11.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f02541ef64077f22bf4924f225c0fd1248c168f86e4b7abdedd87d6ebaceab0f"}, - {file = "pillow-11.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fcb4621042ac4b7865c179bb972ed0da0218a076dc1820ffc48b1d74c1e37fe9"}, - {file = "pillow-11.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:00177a63030d612148e659b55ba99527803288cea7c75fb05766ab7981a8c1b7"}, - {file = "pillow-11.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8853a3bf12afddfdf15f57c4b02d7ded92c7a75a5d7331d19f4f9572a89c17e6"}, - {file = "pillow-11.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3107c66e43bda25359d5ef446f59c497de2b5ed4c7fdba0894f8d6cf3822dafc"}, - {file = "pillow-11.0.0-cp312-cp312-win32.whl", hash = "sha256:86510e3f5eca0ab87429dd77fafc04693195eec7fd6a137c389c3eeb4cfb77c6"}, - {file = "pillow-11.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:8ec4a89295cd6cd4d1058a5e6aec6bf51e0eaaf9714774e1bfac7cfc9051db47"}, - {file = "pillow-11.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:27a7860107500d813fcd203b4ea19b04babe79448268403172782754870dac25"}, - {file = "pillow-11.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bcd1fb5bb7b07f64c15618c89efcc2cfa3e95f0e3bcdbaf4642509de1942a699"}, - {file = "pillow-11.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0e038b0745997c7dcaae350d35859c9715c71e92ffb7e0f4a8e8a16732150f38"}, - {file = "pillow-11.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ae08bd8ffc41aebf578c2af2f9d8749d91f448b3bfd41d7d9ff573d74f2a6b2"}, - {file = "pillow-11.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d69bfd8ec3219ae71bcde1f942b728903cad25fafe3100ba2258b973bd2bc1b2"}, - {file = "pillow-11.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:61b887f9ddba63ddf62fd02a3ba7add935d053b6dd7d58998c630e6dbade8527"}, - {file = "pillow-11.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c6a660307ca9d4867caa8d9ca2c2658ab685de83792d1876274991adec7b93fa"}, - {file = "pillow-11.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:73e3a0200cdda995c7e43dd47436c1548f87a30bb27fb871f352a22ab8dcf45f"}, - {file = "pillow-11.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fba162b8872d30fea8c52b258a542c5dfd7b235fb5cb352240c8d63b414013eb"}, - {file = "pillow-11.0.0-cp313-cp313-win32.whl", hash = "sha256:f1b82c27e89fffc6da125d5eb0ca6e68017faf5efc078128cfaa42cf5cb38798"}, - {file = "pillow-11.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:8ba470552b48e5835f1d23ecb936bb7f71d206f9dfeee64245f30c3270b994de"}, - {file = "pillow-11.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:846e193e103b41e984ac921b335df59195356ce3f71dcfd155aa79c603873b84"}, - {file = "pillow-11.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4ad70c4214f67d7466bea6a08061eba35c01b1b89eaa098040a35272a8efb22b"}, - {file = "pillow-11.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6ec0d5af64f2e3d64a165f490d96368bb5dea8b8f9ad04487f9ab60dc4bb6003"}, - {file = "pillow-11.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c809a70e43c7977c4a42aefd62f0131823ebf7dd73556fa5d5950f5b354087e2"}, - {file = "pillow-11.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:4b60c9520f7207aaf2e1d94de026682fc227806c6e1f55bba7606d1c94dd623a"}, - {file = "pillow-11.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1e2688958a840c822279fda0086fec1fdab2f95bf2b717b66871c4ad9859d7e8"}, - {file = "pillow-11.0.0-cp313-cp313t-win32.whl", hash = "sha256:607bbe123c74e272e381a8d1957083a9463401f7bd01287f50521ecb05a313f8"}, - {file = "pillow-11.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c39ed17edea3bc69c743a8dd3e9853b7509625c2462532e62baa0732163a904"}, - {file = "pillow-11.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:75acbbeb05b86bc53cbe7b7e6fe00fbcf82ad7c684b3ad82e3d711da9ba287d3"}, - {file = "pillow-11.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:2e46773dc9f35a1dd28bd6981332fd7f27bec001a918a72a79b4133cf5291dba"}, - {file = "pillow-11.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2679d2258b7f1192b378e2893a8a0a0ca472234d4c2c0e6bdd3380e8dfa21b6a"}, - {file = "pillow-11.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eda2616eb2313cbb3eebbe51f19362eb434b18e3bb599466a1ffa76a033fb916"}, - {file = "pillow-11.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ec184af98a121fb2da42642dea8a29ec80fc3efbaefb86d8fdd2606619045d"}, - {file = "pillow-11.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:8594f42df584e5b4bb9281799698403f7af489fba84c34d53d1c4bfb71b7c4e7"}, - {file = "pillow-11.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:c12b5ae868897c7338519c03049a806af85b9b8c237b7d675b8c5e089e4a618e"}, - {file = "pillow-11.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:70fbbdacd1d271b77b7721fe3cdd2d537bbbd75d29e6300c672ec6bb38d9672f"}, - {file = "pillow-11.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5178952973e588b3f1360868847334e9e3bf49d19e169bbbdfaf8398002419ae"}, - {file = "pillow-11.0.0-cp39-cp39-win32.whl", hash = "sha256:8c676b587da5673d3c75bd67dd2a8cdfeb282ca38a30f37950511766b26858c4"}, - {file = "pillow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:94f3e1780abb45062287b4614a5bc0874519c86a777d4a7ad34978e86428b8dd"}, - {file = "pillow-11.0.0-cp39-cp39-win_arm64.whl", hash = "sha256:290f2cc809f9da7d6d622550bbf4c1e57518212da51b6a30fe8e0a270a5b78bd"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1187739620f2b365de756ce086fdb3604573337cc28a0d3ac4a01ab6b2d2a6d2"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:fbbcb7b57dc9c794843e3d1258c0fbf0f48656d46ffe9e09b63bbd6e8cd5d0a2"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d203af30149ae339ad1b4f710d9844ed8796e97fda23ffbc4cc472968a47d0b"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a0d3b115009ebb8ac3d2ebec5c2982cc693da935f4ab7bb5c8ebe2f47d36f2"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:73853108f56df97baf2bb8b522f3578221e56f646ba345a372c78326710d3830"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e58876c91f97b0952eb766123bfef372792ab3f4e3e1f1a2267834c2ab131734"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:224aaa38177597bb179f3ec87eeefcce8e4f85e608025e9cfac60de237ba6316"}, - {file = "pillow-11.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5bd2d3bdb846d757055910f0a59792d33b555800813c3b39ada1829c372ccb06"}, - {file = "pillow-11.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:375b8dd15a1f5d2feafff536d47e22f69625c1aa92f12b339ec0b2ca40263273"}, - {file = "pillow-11.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:daffdf51ee5db69a82dd127eabecce20729e21f7a3680cf7cbb23f0829189790"}, - {file = "pillow-11.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7326a1787e3c7b0429659e0a944725e1b03eeaa10edd945a86dead1913383944"}, - {file = "pillow-11.0.0.tar.gz", hash = "sha256:72bacbaf24ac003fea9bff9837d1eedb6088758d41e100c1552930151f677739"}, + {file = "pillow-11.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:e1abe69aca89514737465752b4bcaf8016de61b3be1397a8fc260ba33321b3a8"}, + {file = "pillow-11.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c640e5a06869c75994624551f45e5506e4256562ead981cce820d5ab39ae2192"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a07dba04c5e22824816b2615ad7a7484432d7f540e6fa86af60d2de57b0fcee2"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e267b0ed063341f3e60acd25c05200df4193e15a4a5807075cd71225a2386e26"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bd165131fd51697e22421d0e467997ad31621b74bfc0b75956608cb2906dda07"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:abc56501c3fd148d60659aae0af6ddc149660469082859fa7b066a298bde9482"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:54ce1c9a16a9561b6d6d8cb30089ab1e5eb66918cb47d457bd996ef34182922e"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73ddde795ee9b06257dac5ad42fcb07f3b9b813f8c1f7f870f402f4dc54b5269"}, + {file = "pillow-11.1.0-cp310-cp310-win32.whl", hash = "sha256:3a5fe20a7b66e8135d7fd617b13272626a28278d0e578c98720d9ba4b2439d49"}, + {file = "pillow-11.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:b6123aa4a59d75f06e9dd3dac5bf8bc9aa383121bb3dd9a7a612e05eabc9961a"}, + {file = "pillow-11.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:a76da0a31da6fcae4210aa94fd779c65c75786bc9af06289cd1c184451ef7a65"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e06695e0326d05b06833b40b7ef477e475d0b1ba3a6d27da1bb48c23209bf457"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96f82000e12f23e4f29346e42702b6ed9a2f2fea34a740dd5ffffcc8c539eb35"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3cd561ded2cf2bbae44d4605837221b987c216cff94f49dfeed63488bb228d2"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f189805c8be5ca5add39e6f899e6ce2ed824e65fb45f3c28cb2841911da19070"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dd0052e9db3474df30433f83a71b9b23bd9e4ef1de13d92df21a52c0303b8ab6"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:837060a8599b8f5d402e97197d4924f05a2e0d68756998345c829c33186217b1"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa8dd43daa836b9a8128dbe7d923423e5ad86f50a7a14dc688194b7be5c0dea2"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0a2f91f8a8b367e7a57c6e91cd25af510168091fb89ec5146003e424e1558a96"}, + {file = "pillow-11.1.0-cp311-cp311-win32.whl", hash = "sha256:c12fc111ef090845de2bb15009372175d76ac99969bdf31e2ce9b42e4b8cd88f"}, + {file = "pillow-11.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbd43429d0d7ed6533b25fc993861b8fd512c42d04514a0dd6337fb3ccf22761"}, + {file = "pillow-11.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:f7955ecf5609dee9442cbface754f2c6e541d9e6eda87fad7f7a989b0bdb9d71"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2062ffb1d36544d42fcaa277b069c88b01bb7298f4efa06731a7fd6cc290b81a"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a85b653980faad27e88b141348707ceeef8a1186f75ecc600c395dcac19f385b"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9409c080586d1f683df3f184f20e36fb647f2e0bc3988094d4fd8c9f4eb1b3b3"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fdadc077553621911f27ce206ffcbec7d3f8d7b50e0da39f10997e8e2bb7f6a"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:93a18841d09bcdd774dcdc308e4537e1f867b3dec059c131fde0327899734aa1"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9aa9aeddeed452b2f616ff5507459e7bab436916ccb10961c4a382cd3e03f47f"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cdcdb0b896e981678eee140d882b70092dac83ac1cdf6b3a60e2216a73f2b91"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36ba10b9cb413e7c7dfa3e189aba252deee0602c86c309799da5a74009ac7a1c"}, + {file = "pillow-11.1.0-cp312-cp312-win32.whl", hash = "sha256:cfd5cd998c2e36a862d0e27b2df63237e67273f2fc78f47445b14e73a810e7e6"}, + {file = "pillow-11.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a697cd8ba0383bba3d2d3ada02b34ed268cb548b369943cd349007730c92bddf"}, + {file = "pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae98e14432d458fc3de11a77ccb3ae65ddce70f730e7c76140653048c71bfcbc"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cc1331b6d5a6e144aeb5e626f4375f5b7ae9934ba620c0ac6b3e43d5e683a0f0"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:758e9d4ef15d3560214cddbc97b8ef3ef86ce04d62ddac17ad39ba87e89bd3b1"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b523466b1a31d0dcef7c5be1f20b942919b62fd6e9a9be199d035509cbefc0ec"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9044b5e4f7083f209c4e35aa5dd54b1dd5b112b108648f5c902ad586d4f945c5"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3764d53e09cdedd91bee65c2527815d315c6b90d7b8b79759cc48d7bf5d4f114"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:31eba6bbdd27dde97b0174ddf0297d7a9c3a507a8a1480e1e60ef914fe23d352"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5d658fbd9f0d6eea113aea286b21d3cd4d3fd978157cbf2447a6035916506d3"}, + {file = "pillow-11.1.0-cp313-cp313-win32.whl", hash = "sha256:f86d3a7a9af5d826744fabf4afd15b9dfef44fe69a98541f666f66fbb8d3fef9"}, + {file = "pillow-11.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:593c5fd6be85da83656b93ffcccc2312d2d149d251e98588b14fbc288fd8909c"}, + {file = "pillow-11.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:11633d58b6ee5733bde153a8dafd25e505ea3d32e261accd388827ee987baf65"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:70ca5ef3b3b1c4a0812b5c63c57c23b63e53bc38e758b37a951e5bc466449861"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8000376f139d4d38d6851eb149b321a52bb8893a88dae8ee7d95840431977081"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ee85f0696a17dd28fbcfceb59f9510aa71934b483d1f5601d1030c3c8304f3c"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:dd0e081319328928531df7a0e63621caf67652c8464303fd102141b785ef9547"}, + {file = "pillow-11.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e63e4e5081de46517099dc30abe418122f54531a6ae2ebc8680bcd7096860eab"}, + {file = "pillow-11.1.0-cp313-cp313t-win32.whl", hash = "sha256:dda60aa465b861324e65a78c9f5cf0f4bc713e4309f83bc387be158b077963d9"}, + {file = "pillow-11.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ad5db5781c774ab9a9b2c4302bbf0c1014960a0a7be63278d13ae6fdf88126fe"}, + {file = "pillow-11.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:67cd427c68926108778a9005f2a04adbd5e67c442ed21d95389fe1d595458756"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:bf902d7413c82a1bfa08b06a070876132a5ae6b2388e2712aab3a7cbc02205c6"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c1eec9d950b6fe688edee07138993e54ee4ae634c51443cfb7c1e7613322718e"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e275ee4cb11c262bd108ab2081f750db2a1c0b8c12c1897f27b160c8bd57bbc"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4db853948ce4e718f2fc775b75c37ba2efb6aaea41a1a5fc57f0af59eee774b2"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ab8a209b8485d3db694fa97a896d96dd6533d63c22829043fd9de627060beade"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:54251ef02a2309b5eec99d151ebf5c9904b77976c8abdcbce7891ed22df53884"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5bb94705aea800051a743aa4874bb1397d4695fb0583ba5e425ee0328757f196"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89dbdb3e6e9594d512780a5a1c42801879628b38e3efc7038094430844e271d8"}, + {file = "pillow-11.1.0-cp39-cp39-win32.whl", hash = "sha256:e5449ca63da169a2e6068dd0e2fcc8d91f9558aba89ff6d02121ca8ab11e79e5"}, + {file = "pillow-11.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:3362c6ca227e65c54bf71a5f88b3d4565ff1bcbc63ae72c34b07bbb1cc59a43f"}, + {file = "pillow-11.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:b20be51b37a75cc54c2c55def3fa2c65bb94ba859dde241cd0a4fd302de5ae0a"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8c730dc3a83e5ac137fbc92dfcfe1511ce3b2b5d7578315b63dbbb76f7f51d90"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7d33d2fae0e8b170b6a6c57400e077412240f6f5bb2a342cf1ee512a787942bb"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8d65b38173085f24bc07f8b6c505cbb7418009fa1a1fcb111b1f4961814a442"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:015c6e863faa4779251436db398ae75051469f7c903b043a48f078e437656f83"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d44ff19eea13ae4acdaaab0179fa68c0c6f2f45d66a4d8ec1eda7d6cecbcc15f"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3d8da4a631471dfaf94c10c85f5277b1f8e42ac42bade1ac67da4b4a7359b73"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4637b88343166249fe8aa94e7c4a62a180c4b3898283bb5d3d2fd5fe10d8e4e0"}, + {file = "pillow-11.1.0.tar.gz", hash = "sha256:368da70808b36d73b4b390a8ffac11069f8a5c85f29eff1f1b01bcf3ef5b2a20"}, ] [package.extras] docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] fpx = ["olefile"] mic = ["olefile"] -tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"] typing = ["typing-extensions"] xmp = ["defusedxml"] @@ -1649,18 +1646,18 @@ type = ["mypy (>=1.11.2)"] [[package]] name = "playwright" -version = "1.48.0" +version = "1.49.1" description = "A high-level API to automate web browsers" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "playwright-1.48.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:082bce2739f1078acc7d0734da8cc0e23eb91b7fae553f3316d733276f09a6b1"}, - {file = "playwright-1.48.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7da2eb51a19c7f3b523e9faa9d98e7af92e52eb983a099979ea79c9668e3cbf7"}, - {file = "playwright-1.48.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:115b988d1da322358b77bc3bf2d3cc90f8c881e691461538e7df91614c4833c9"}, - {file = "playwright-1.48.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:8dabb80e62f667fe2640a8b694e26a7b884c0b4803f7514a3954fc849126227b"}, - {file = "playwright-1.48.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ff8303409ebed76bed4c3d655340320b768817d900ba208b394fdd7d7939a5c"}, - {file = "playwright-1.48.0-py3-none-win32.whl", hash = "sha256:85598c360c590076d4f435525be991246d74a905b654ac19d26eab7ed9b98b2d"}, - {file = "playwright-1.48.0-py3-none-win_amd64.whl", hash = "sha256:e0e87b0c4dc8fce83c725dd851aec37bc4e882bb225ec8a96bd83cf32d4f1623"}, + {file = "playwright-1.49.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:1041ffb45a0d0bc44d698d3a5aa3ac4b67c9bd03540da43a0b70616ad52592b8"}, + {file = "playwright-1.49.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9f38ed3d0c1f4e0a6d1c92e73dd9a61f8855133249d6f0cec28648d38a7137be"}, + {file = "playwright-1.49.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:3be48c6d26dc819ca0a26567c1ae36a980a0303dcd4249feb6f59e115aaddfb8"}, + {file = "playwright-1.49.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:753ca90ee31b4b03d165cfd36e477309ebf2b4381953f2a982ff612d85b147d2"}, + {file = "playwright-1.49.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd9bc8dab37aa25198a01f555f0a2e2c3813fe200fef018ac34dfe86b34994b9"}, + {file = "playwright-1.49.1-py3-none-win32.whl", hash = "sha256:43b304be67f096058e587dac453ece550eff87b8fbed28de30f4f022cc1745bb"}, + {file = "playwright-1.49.1-py3-none-win_amd64.whl", hash = "sha256:47b23cb346283278f5b4d1e1990bcb6d6302f80c0aa0ca93dd0601a1400191df"}, ] [package.dependencies] @@ -1703,109 +1700,93 @@ poetry-plugin = ["poetry (>=1.0,<2.0)"] [[package]] name = "propcache" -version = "0.2.0" +version = "0.2.1" description = "Accelerated property cache" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"}, - {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"}, - {file = "propcache-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:33ac8f098df0585c0b53009f039dfd913b38c1d2edafed0cedcc0c32a05aa110"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e48e8875e6c13909c800fa344cd54cc4b2b0db1d5f911f840458a500fde2c2"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388f3217649d6d59292b722d940d4d2e1e6a7003259eb835724092a1cca0203a"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f571aea50ba5623c308aa146eb650eebf7dbe0fd8c5d946e28343cb3b5aad577"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3dfafb44f7bb35c0c06eda6b2ab4bfd58f02729e7c4045e179f9a861b07c9850"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3ebe9a75be7ab0b7da2464a77bb27febcb4fab46a34f9288f39d74833db7f61"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d2f0d0f976985f85dfb5f3d685697ef769faa6b71993b46b295cdbbd6be8cc37"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a3dc1a4b165283bd865e8f8cb5f0c64c05001e0718ed06250d8cac9bec115b48"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9e0f07b42d2a50c7dd2d8675d50f7343d998c64008f1da5fef888396b7f84630"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e63e3e1e0271f374ed489ff5ee73d4b6e7c60710e1f76af5f0e1a6117cd26394"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:56bb5c98f058a41bb58eead194b4db8c05b088c93d94d5161728515bd52b052b"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7665f04d0c7f26ff8bb534e1c65068409bf4687aa2534faf7104d7182debb336"}, - {file = "propcache-0.2.0-cp310-cp310-win32.whl", hash = "sha256:7cf18abf9764746b9c8704774d8b06714bcb0a63641518a3a89c7f85cc02c2ad"}, - {file = "propcache-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:cfac69017ef97db2438efb854edf24f5a29fd09a536ff3a992b75990720cdc99"}, - {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:63f13bf09cc3336eb04a837490b8f332e0db41da66995c9fd1ba04552e516354"}, - {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608cce1da6f2672a56b24a015b42db4ac612ee709f3d29f27a00c943d9e851de"}, - {file = "propcache-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:466c219deee4536fbc83c08d09115249db301550625c7fef1c5563a584c9bc87"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2db02409338bf36590aa985a461b2c96fce91f8e7e0f14c50c5fcc4f229016"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a6ed8db0a556343d566a5c124ee483ae113acc9a557a807d439bcecc44e7dfbb"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91997d9cb4a325b60d4e3f20967f8eb08dfcb32b22554d5ef78e6fd1dda743a2"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c7dde9e533c0a49d802b4f3f218fa9ad0a1ce21f2c2eb80d5216565202acab4"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffcad6c564fe6b9b8916c1aefbb37a362deebf9394bd2974e9d84232e3e08504"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:97a58a28bcf63284e8b4d7b460cbee1edaab24634e82059c7b8c09e65284f178"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:945db8ee295d3af9dbdbb698cce9bbc5c59b5c3fe328bbc4387f59a8a35f998d"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:39e104da444a34830751715f45ef9fc537475ba21b7f1f5b0f4d71a3b60d7fe2"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c5ecca8f9bab618340c8e848d340baf68bcd8ad90a8ecd7a4524a81c1764b3db"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c436130cc779806bdf5d5fae0d848713105472b8566b75ff70048c47d3961c5b"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:191db28dc6dcd29d1a3e063c3be0b40688ed76434622c53a284e5427565bbd9b"}, - {file = "propcache-0.2.0-cp311-cp311-win32.whl", hash = "sha256:5f2564ec89058ee7c7989a7b719115bdfe2a2fb8e7a4543b8d1c0cc4cf6478c1"}, - {file = "propcache-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e2e54267980349b723cff366d1e29b138b9a60fa376664a157a342689553f71"}, - {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2"}, - {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7"}, - {file = "propcache-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2e900bad2a8456d00a113cad8c13343f3b1f327534e3589acc2219729237a2e8"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f52a68c21363c45297aca15561812d542f8fc683c85201df0bebe209e349f793"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e41d67757ff4fbc8ef2af99b338bfb955010444b92929e9e55a6d4dcc3c4f09"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a64e32f8bd94c105cc27f42d3b658902b5bcc947ece3c8fe7bc1b05982f60e89"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55346705687dbd7ef0d77883ab4f6fabc48232f587925bdaf95219bae072491e"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6994984550eaf25dd7fc7bd1b700ff45c894149341725bb4edc67f0ffa94efa4"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:56295eb1e5f3aecd516d91b00cfd8bf3a13991de5a479df9e27dd569ea23959c"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:439e76255daa0f8151d3cb325f6dd4a3e93043e6403e6491813bcaaaa8733887"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f6475a1b2ecb310c98c28d271a30df74f9dd436ee46d09236a6b750a7599ce57"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3444cdba6628accf384e349014084b1cacd866fbb88433cd9d279d90a54e0b23"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a9d9b4d0a9b38d1c391bb4ad24aa65f306c6f01b512e10a8a34a2dc5675d348"}, - {file = "propcache-0.2.0-cp312-cp312-win32.whl", hash = "sha256:69d3a98eebae99a420d4b28756c8ce6ea5a29291baf2dc9ff9414b42676f61d5"}, - {file = "propcache-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ad9c9b99b05f163109466638bd30ada1722abb01bbb85c739c50b6dc11f92dc3"}, - {file = "propcache-0.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ecddc221a077a8132cf7c747d5352a15ed763b674c0448d811f408bf803d9ad7"}, - {file = "propcache-0.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0e53cb83fdd61cbd67202735e6a6687a7b491c8742dfc39c9e01e80354956763"}, - {file = "propcache-0.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92fe151145a990c22cbccf9ae15cae8ae9eddabfc949a219c9f667877e40853d"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a21ef516d36909931a2967621eecb256018aeb11fc48656e3257e73e2e247a"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f88a4095e913f98988f5b338c1d4d5d07dbb0b6bad19892fd447484e483ba6b"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a5b3bb545ead161be780ee85a2b54fdf7092815995661947812dde94a40f6fb"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67aeb72e0f482709991aa91345a831d0b707d16b0257e8ef88a2ad246a7280bf"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c997f8c44ec9b9b0bcbf2d422cc00a1d9b9c681f56efa6ca149a941e5560da2"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2a66df3d4992bc1d725b9aa803e8c5a66c010c65c741ad901e260ece77f58d2f"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3ebbcf2a07621f29638799828b8d8668c421bfb94c6cb04269130d8de4fb7136"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1235c01ddaa80da8235741e80815ce381c5267f96cc49b1477fdcf8c047ef325"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3947483a381259c06921612550867b37d22e1df6d6d7e8361264b6d037595f44"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d5bed7f9805cc29c780f3aee05de3262ee7ce1f47083cfe9f77471e9d6777e83"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e4a91d44379f45f5e540971d41e4626dacd7f01004826a18cb048e7da7e96544"}, - {file = "propcache-0.2.0-cp313-cp313-win32.whl", hash = "sha256:f902804113e032e2cdf8c71015651c97af6418363bea8d78dc0911d56c335032"}, - {file = "propcache-0.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8f188cfcc64fb1266f4684206c9de0e80f54622c3f22a910cbd200478aeae61e"}, - {file = "propcache-0.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:53d1bd3f979ed529f0805dd35ddaca330f80a9a6d90bc0121d2ff398f8ed8861"}, - {file = "propcache-0.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:83928404adf8fb3d26793665633ea79b7361efa0287dfbd372a7e74311d51ee6"}, - {file = "propcache-0.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77a86c261679ea5f3896ec060be9dc8e365788248cc1e049632a1be682442063"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:218db2a3c297a3768c11a34812e63b3ac1c3234c3a086def9c0fee50d35add1f"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7735e82e3498c27bcb2d17cb65d62c14f1100b71723b68362872bca7d0913d90"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20a617c776f520c3875cf4511e0d1db847a076d720714ae35ffe0df3e440be68"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67b69535c870670c9f9b14a75d28baa32221d06f6b6fa6f77a0a13c5a7b0a5b9"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4569158070180c3855e9c0791c56be3ceeb192defa2cdf6a3f39e54319e56b89"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:db47514ffdbd91ccdc7e6f8407aac4ee94cc871b15b577c1c324236b013ddd04"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:2a60ad3e2553a74168d275a0ef35e8c0a965448ffbc3b300ab3a5bb9956c2162"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:662dd62358bdeaca0aee5761de8727cfd6861432e3bb828dc2a693aa0471a563"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:25a1f88b471b3bc911d18b935ecb7115dff3a192b6fef46f0bfaf71ff4f12418"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:f60f0ac7005b9f5a6091009b09a419ace1610e163fa5deaba5ce3484341840e7"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:74acd6e291f885678631b7ebc85d2d4aec458dd849b8c841b57ef04047833bed"}, - {file = "propcache-0.2.0-cp38-cp38-win32.whl", hash = "sha256:d9b6ddac6408194e934002a69bcaadbc88c10b5f38fb9307779d1c629181815d"}, - {file = "propcache-0.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:676135dcf3262c9c5081cc8f19ad55c8a64e3f7282a21266d05544450bffc3a5"}, - {file = "propcache-0.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:25c8d773a62ce0451b020c7b29a35cfbc05de8b291163a7a0f3b7904f27253e6"}, - {file = "propcache-0.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:375a12d7556d462dc64d70475a9ee5982465fbb3d2b364f16b86ba9135793638"}, - {file = "propcache-0.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1ec43d76b9677637a89d6ab86e1fef70d739217fefa208c65352ecf0282be957"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f45eec587dafd4b2d41ac189c2156461ebd0c1082d2fe7013571598abb8505d1"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc092ba439d91df90aea38168e11f75c655880c12782facf5cf9c00f3d42b562"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa1076244f54bb76e65e22cb6910365779d5c3d71d1f18b275f1dfc7b0d71b4d"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:682a7c79a2fbf40f5dbb1eb6bfe2cd865376deeac65acf9beb607505dced9e12"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e40876731f99b6f3c897b66b803c9e1c07a989b366c6b5b475fafd1f7ba3fb8"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:363ea8cd3c5cb6679f1c2f5f1f9669587361c062e4899fce56758efa928728f8"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:140fbf08ab3588b3468932974a9331aff43c0ab8a2ec2c608b6d7d1756dbb6cb"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e70fac33e8b4ac63dfc4c956fd7d85a0b1139adcfc0d964ce288b7c527537fea"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b33d7a286c0dc1a15f5fc864cc48ae92a846df287ceac2dd499926c3801054a6"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f6d5749fdd33d90e34c2efb174c7e236829147a2713334d708746e94c4bde40d"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22aa8f2272d81d9317ff5756bb108021a056805ce63dd3630e27d042c8092798"}, - {file = "propcache-0.2.0-cp39-cp39-win32.whl", hash = "sha256:73e4b40ea0eda421b115248d7e79b59214411109a5bc47d0d48e4c73e3b8fcf9"}, - {file = "propcache-0.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:9517d5e9e0731957468c29dbfd0f976736a0e55afaea843726e887f36fe017df"}, - {file = "propcache-0.2.0-py3-none-any.whl", hash = "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036"}, - {file = "propcache-0.2.0.tar.gz", hash = "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70"}, + {file = "propcache-0.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6b3f39a85d671436ee3d12c017f8fdea38509e4f25b28eb25877293c98c243f6"}, + {file = "propcache-0.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d51fbe4285d5db5d92a929e3e21536ea3dd43732c5b177c7ef03f918dff9f2"}, + {file = "propcache-0.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6445804cf4ec763dc70de65a3b0d9954e868609e83850a47ca4f0cb64bd79fea"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9479aa06a793c5aeba49ce5c5692ffb51fcd9a7016e017d555d5e2b0045d212"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9631c5e8b5b3a0fda99cb0d29c18133bca1e18aea9effe55adb3da1adef80d3"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3156628250f46a0895f1f36e1d4fbe062a1af8718ec3ebeb746f1d23f0c5dc4d"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6fb63ae352e13748289f04f37868099e69dba4c2b3e271c46061e82c745634"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:887d9b0a65404929641a9fabb6452b07fe4572b269d901d622d8a34a4e9043b2"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a96dc1fa45bd8c407a0af03b2d5218392729e1822b0c32e62c5bf7eeb5fb3958"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a7e65eb5c003a303b94aa2c3852ef130230ec79e349632d030e9571b87c4698c"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:999779addc413181912e984b942fbcc951be1f5b3663cd80b2687758f434c583"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:19a0f89a7bb9d8048d9c4370c9c543c396e894c76be5525f5e1ad287f1750ddf"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1ac2f5fe02fa75f56e1ad473f1175e11f475606ec9bd0be2e78e4734ad575034"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:574faa3b79e8ebac7cb1d7930f51184ba1ccf69adfdec53a12f319a06030a68b"}, + {file = "propcache-0.2.1-cp310-cp310-win32.whl", hash = "sha256:03ff9d3f665769b2a85e6157ac8b439644f2d7fd17615a82fa55739bc97863f4"}, + {file = "propcache-0.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:2d3af2e79991102678f53e0dbf4c35de99b6b8b58f29a27ca0325816364caaba"}, + {file = "propcache-0.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1ffc3cca89bb438fb9c95c13fc874012f7b9466b89328c3c8b1aa93cdcfadd16"}, + {file = "propcache-0.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f174bbd484294ed9fdf09437f889f95807e5f229d5d93588d34e92106fbf6717"}, + {file = "propcache-0.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:70693319e0b8fd35dd863e3e29513875eb15c51945bf32519ef52927ca883bc3"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b480c6a4e1138e1aa137c0079b9b6305ec6dcc1098a8ca5196283e8a49df95a9"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d27b84d5880f6d8aa9ae3edb253c59d9f6642ffbb2c889b78b60361eed449787"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:857112b22acd417c40fa4595db2fe28ab900c8c5fe4670c7989b1c0230955465"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf6c4150f8c0e32d241436526f3c3f9cbd34429492abddbada2ffcff506c51af"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66d4cfda1d8ed687daa4bc0274fcfd5267873db9a5bc0418c2da19273040eeb7"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2f992c07c0fca81655066705beae35fc95a2fa7366467366db627d9f2ee097f"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:4a571d97dbe66ef38e472703067021b1467025ec85707d57e78711c085984e54"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:bb6178c241278d5fe853b3de743087be7f5f4c6f7d6d22a3b524d323eecec505"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ad1af54a62ffe39cf34db1aa6ed1a1873bd548f6401db39d8e7cd060b9211f82"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e7048abd75fe40712005bcfc06bb44b9dfcd8e101dda2ecf2f5aa46115ad07ca"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:160291c60081f23ee43d44b08a7e5fb76681221a8e10b3139618c5a9a291b84e"}, + {file = "propcache-0.2.1-cp311-cp311-win32.whl", hash = "sha256:819ce3b883b7576ca28da3861c7e1a88afd08cc8c96908e08a3f4dd64a228034"}, + {file = "propcache-0.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:edc9fc7051e3350643ad929df55c451899bb9ae6d24998a949d2e4c87fb596d3"}, + {file = "propcache-0.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:081a430aa8d5e8876c6909b67bd2d937bfd531b0382d3fdedb82612c618bc41a"}, + {file = "propcache-0.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d2ccec9ac47cf4e04897619c0e0c1a48c54a71bdf045117d3a26f80d38ab1fb0"}, + {file = "propcache-0.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:14d86fe14b7e04fa306e0c43cdbeebe6b2c2156a0c9ce56b815faacc193e320d"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:049324ee97bb67285b49632132db351b41e77833678432be52bdd0289c0e05e4"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1cd9a1d071158de1cc1c71a26014dcdfa7dd3d5f4f88c298c7f90ad6f27bb46d"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98110aa363f1bb4c073e8dcfaefd3a5cea0f0834c2aab23dda657e4dab2f53b5"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:647894f5ae99c4cf6bb82a1bb3a796f6e06af3caa3d32e26d2350d0e3e3faf24"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfd3223c15bebe26518d58ccf9a39b93948d3dcb3e57a20480dfdd315356baff"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d71264a80f3fcf512eb4f18f59423fe82d6e346ee97b90625f283df56aee103f"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e73091191e4280403bde6c9a52a6999d69cdfde498f1fdf629105247599b57ec"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3935bfa5fede35fb202c4b569bb9c042f337ca4ff7bd540a0aa5e37131659348"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f508b0491767bb1f2b87fdfacaba5f7eddc2f867740ec69ece6d1946d29029a6"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1672137af7c46662a1c2be1e8dc78cb6d224319aaa40271c9257d886be4363a6"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b74c261802d3d2b85c9df2dfb2fa81b6f90deeef63c2db9f0e029a3cac50b518"}, + {file = "propcache-0.2.1-cp312-cp312-win32.whl", hash = "sha256:d09c333d36c1409d56a9d29b3a1b800a42c76a57a5a8907eacdbce3f18768246"}, + {file = "propcache-0.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:c214999039d4f2a5b2073ac506bba279945233da8c786e490d411dfc30f855c1"}, + {file = "propcache-0.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aca405706e0b0a44cc6bfd41fbe89919a6a56999157f6de7e182a990c36e37bc"}, + {file = "propcache-0.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:12d1083f001ace206fe34b6bdc2cb94be66d57a850866f0b908972f90996b3e9"}, + {file = "propcache-0.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d93f3307ad32a27bda2e88ec81134b823c240aa3abb55821a8da553eed8d9439"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba278acf14471d36316159c94a802933d10b6a1e117b8554fe0d0d9b75c9d536"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e6281aedfca15301c41f74d7005e6e3f4ca143584ba696ac69df4f02f40d629"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b750a8e5a1262434fb1517ddf64b5de58327f1adc3524a5e44c2ca43305eb0b"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf72af5e0fb40e9babf594308911436c8efde3cb5e75b6f206c34ad18be5c052"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2d0a12018b04f4cb820781ec0dffb5f7c7c1d2a5cd22bff7fb055a2cb19ebce"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e800776a79a5aabdb17dcc2346a7d66d0777e942e4cd251defeb084762ecd17d"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:4160d9283bd382fa6c0c2b5e017acc95bc183570cd70968b9202ad6d8fc48dce"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:30b43e74f1359353341a7adb783c8f1b1c676367b011709f466f42fda2045e95"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:58791550b27d5488b1bb52bc96328456095d96206a250d28d874fafe11b3dfaf"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0f022d381747f0dfe27e99d928e31bc51a18b65bb9e481ae0af1380a6725dd1f"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:297878dc9d0a334358f9b608b56d02e72899f3b8499fc6044133f0d319e2ec30"}, + {file = "propcache-0.2.1-cp313-cp313-win32.whl", hash = "sha256:ddfab44e4489bd79bda09d84c430677fc7f0a4939a73d2bba3073036f487a0a6"}, + {file = "propcache-0.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:556fc6c10989f19a179e4321e5d678db8eb2924131e64652a51fe83e4c3db0e1"}, + {file = "propcache-0.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6a9a8c34fb7bb609419a211e59da8887eeca40d300b5ea8e56af98f6fbbb1541"}, + {file = "propcache-0.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ae1aa1cd222c6d205853b3013c69cd04515f9d6ab6de4b0603e2e1c33221303e"}, + {file = "propcache-0.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:accb6150ce61c9c4b7738d45550806aa2b71c7668c6942f17b0ac182b6142fd4"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5eee736daafa7af6d0a2dc15cc75e05c64f37fc37bafef2e00d77c14171c2097"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7a31fc1e1bd362874863fdeed71aed92d348f5336fd84f2197ba40c59f061bd"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba4cfa1052819d16699e1d55d18c92b6e094d4517c41dd231a8b9f87b6fa681"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f089118d584e859c62b3da0892b88a83d611c2033ac410e929cb6754eec0ed16"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:781e65134efaf88feb447e8c97a51772aa75e48b794352f94cb7ea717dedda0d"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31f5af773530fd3c658b32b6bdc2d0838543de70eb9a2156c03e410f7b0d3aae"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:a7a078f5d37bee6690959c813977da5291b24286e7b962e62a94cec31aa5188b"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:cea7daf9fc7ae6687cf1e2c049752f19f146fdc37c2cc376e7d0032cf4f25347"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:8b3489ff1ed1e8315674d0775dc7d2195fb13ca17b3808721b54dbe9fd020faf"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9403db39be1393618dd80c746cb22ccda168efce239c73af13c3763ef56ffc04"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5d97151bc92d2b2578ff7ce779cdb9174337390a535953cbb9452fb65164c587"}, + {file = "propcache-0.2.1-cp39-cp39-win32.whl", hash = "sha256:9caac6b54914bdf41bcc91e7eb9147d331d29235a7c967c150ef5df6464fd1bb"}, + {file = "propcache-0.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:92fc4500fcb33899b05ba73276dfb684a20d31caa567b7cb5252d48f896a91b1"}, + {file = "propcache-0.2.1-py3-none-any.whl", hash = "sha256:52277518d6aae65536e9cea52d4e7fd2f7a66f4aa2d30ed3f2fcea620ace3c54"}, + {file = "propcache-0.2.1.tar.gz", hash = "sha256:3f77ce728b19cb537714499928fe800c3dda29e8d9428778fc7c186da4c09a64"}, ] [[package]] @@ -1821,22 +1802,19 @@ files = [ [[package]] name = "pydantic" -version = "2.9.2" +version = "2.10.4" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, - {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, + {file = "pydantic-2.10.4-py3-none-any.whl", hash = "sha256:597e135ea68be3a37552fb524bc7d0d66dcf93d395acd93a00682f1efcb8ee3d"}, + {file = "pydantic-2.10.4.tar.gz", hash = "sha256:82f12e9723da6de4fe2ba888b5971157b3be7ad914267dea8f05f82b28254f06"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.23.4" -typing-extensions = [ - {version = ">=4.6.1", markers = "python_version < \"3.13\""}, - {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, -] +pydantic-core = "2.27.2" +typing-extensions = ">=4.12.2" [package.extras] email = ["email-validator (>=2.0.0)"] @@ -1844,100 +1822,111 @@ timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.23.4" +version = "2.27.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, - {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, - {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, - {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, - {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, - {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, - {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, - {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, - {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, - {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, - {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, - {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, - {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, - {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, + {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"}, + {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af"}, + {file = "pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4"}, + {file = "pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31"}, + {file = "pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc"}, + {file = "pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0"}, + {file = "pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b"}, + {file = "pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b"}, + {file = "pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b"}, + {file = "pydantic_core-2.27.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d3e8d504bdd3f10835468f29008d72fc8359d95c9c415ce6e767203db6127506"}, + {file = "pydantic_core-2.27.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:521eb9b7f036c9b6187f0b47318ab0d7ca14bd87f776240b90b21c1f4f149320"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85210c4d99a0114f5a9481b44560d7d1e35e32cc5634c656bc48e590b669b145"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d716e2e30c6f140d7560ef1538953a5cd1a87264c737643d481f2779fc247fe1"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f66d89ba397d92f840f8654756196d93804278457b5fbede59598a1f9f90b228"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:669e193c1c576a58f132e3158f9dfa9662969edb1a250c54d8fa52590045f046"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdbe7629b996647b99c01b37f11170a57ae675375b14b8c13b8518b8320ced5"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d262606bf386a5ba0b0af3b97f37c83d7011439e3dc1a9298f21efb292e42f1a"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cabb9bcb7e0d97f74df8646f34fc76fbf793b7f6dc2438517d7a9e50eee4f14d"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d2d63f1215638d28221f664596b1ccb3944f6e25dd18cd3b86b0a4c408d5ebb9"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bca101c00bff0adb45a833f8451b9105d9df18accb8743b08107d7ada14bd7da"}, + {file = "pydantic_core-2.27.2-cp38-cp38-win32.whl", hash = "sha256:f6f8e111843bbb0dee4cb6594cdc73e79b3329b526037ec242a3e49012495b3b"}, + {file = "pydantic_core-2.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:fd1aea04935a508f62e0d0ef1f5ae968774a32afc306fb8545e06f5ff5cdf3ad"}, + {file = "pydantic_core-2.27.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c10eb4f1659290b523af58fa7cffb452a61ad6ae5613404519aee4bfbf1df993"}, + {file = "pydantic_core-2.27.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef592d4bad47296fb11f96cd7dc898b92e795032b4894dfb4076cfccd43a9308"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61709a844acc6bf0b7dce7daae75195a10aac96a596ea1b776996414791ede4"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c5f762659e47fdb7b16956c71598292f60a03aa92f8b6351504359dbdba6cf"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c9775e339e42e79ec99c441d9730fccf07414af63eac2f0e48e08fd38a64d76"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57762139821c31847cfb2df63c12f725788bd9f04bc2fb392790959b8f70f118"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d1e85068e818c73e048fe28cfc769040bb1f475524f4745a5dc621f75ac7630"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:097830ed52fd9e427942ff3b9bc17fab52913b2f50f2880dc4a5611446606a54"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044a50963a614ecfae59bb1eaf7ea7efc4bc62f49ed594e18fa1e5d953c40e9f"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:4e0b4220ba5b40d727c7f879eac379b822eee5d8fff418e9d3381ee45b3b0362"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4f4bb20d75e9325cc9696c6802657b58bc1dbbe3022f32cc2b2b632c3fbb96"}, + {file = "pydantic_core-2.27.2-cp39-cp39-win32.whl", hash = "sha256:cca63613e90d001b9f2f9a9ceb276c308bfa2a43fafb75c8031c4f66039e8c6e"}, + {file = "pydantic_core-2.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:77d1bca19b0f7021b3a982e6f903dcd5b2b06076def36a652e3907f596e29f67"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c33939a82924da9ed65dab5a65d427205a73181d8098e79b6b426bdf8ad4e656"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00bad2484fa6bda1e216e7345a798bd37c68fb2d97558edd584942aa41b7d278"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817e2b40aba42bac6f457498dacabc568c3b7a986fc9ba7c8d9d260b71485fb"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:251136cdad0cb722e93732cb45ca5299fb56e1344a833640bf93b2803f8d1bfd"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2088237af596f0a524d3afc39ab3b036e8adb054ee57cbb1dcf8e09da5b29cc"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d4041c0b966a84b4ae7a09832eb691a35aec90910cd2dbe7a208de59be77965b"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8083d4e875ebe0b864ffef72a4304827015cff328a1be6e22cc850753bfb122b"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f141ee28a0ad2123b6611b6ceff018039df17f32ada8b534e6aa039545a3efb2"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7d0c8399fcc1848491f00e0314bd59fb34a9c008761bcb422a057670c3f65e35"}, + {file = "pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39"}, ] [package.dependencies] @@ -1973,17 +1962,17 @@ files = [ [[package]] name = "pylint" -version = "3.3.1" +version = "3.3.3" description = "python code static checker" optional = false python-versions = ">=3.9.0" files = [ - {file = "pylint-3.3.1-py3-none-any.whl", hash = "sha256:2f846a466dd023513240bc140ad2dd73bfc080a5d85a710afdb728c420a5a2b9"}, - {file = "pylint-3.3.1.tar.gz", hash = "sha256:9f3dcc87b1203e612b78d91a896407787e708b3f189b5fa0b307712d49ff0c6e"}, + {file = "pylint-3.3.3-py3-none-any.whl", hash = "sha256:26e271a2bc8bce0fc23833805a9076dd9b4d5194e2a02164942cb3cdc37b4183"}, + {file = "pylint-3.3.3.tar.gz", hash = "sha256:07c607523b17e6d16e2ae0d7ef59602e332caa762af64203c24b41c27139f36a"}, ] [package.dependencies] -astroid = ">=3.3.4,<=3.4.0-dev0" +astroid = ">=3.3.8,<=3.4.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.2", markers = "python_version < \"3.11\""}, @@ -2017,12 +2006,12 @@ pylint = ">=1.7" [[package]] name = "pylint-pydantic" -version = "0.3.2" +version = "0.3.4" description = "A Pylint plugin to help Pylint understand the Pydantic" optional = false python-versions = ">=3.8" files = [ - {file = "pylint_pydantic-0.3.2-py3-none-any.whl", hash = "sha256:e5cec02370aa68ac8eff138e5d573b0ac049bab864e9a6c3a9057cf043440aa1"}, + {file = "pylint_pydantic-0.3.4-py3-none-any.whl", hash = "sha256:f82fdf6b05045102fef2bd8b553a118aadf80155116f374a76eb201c47160a68"}, ] [package.dependencies] @@ -2032,13 +2021,13 @@ pylint-plugin-utils = "*" [[package]] name = "pytest" -version = "8.3.3" +version = "8.3.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, - {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, + {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, + {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, ] [package.dependencies] @@ -2301,112 +2290,125 @@ requests = ">=1.0.0" [[package]] name = "rpds-py" -version = "0.21.0" +version = "0.22.3" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.9" files = [ - {file = "rpds_py-0.21.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a017f813f24b9df929674d0332a374d40d7f0162b326562daae8066b502d0590"}, - {file = "rpds_py-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:20cc1ed0bcc86d8e1a7e968cce15be45178fd16e2ff656a243145e0b439bd250"}, - {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad116dda078d0bc4886cb7840e19811562acdc7a8e296ea6ec37e70326c1b41c"}, - {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:808f1ac7cf3b44f81c9475475ceb221f982ef548e44e024ad5f9e7060649540e"}, - {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de552f4a1916e520f2703ec474d2b4d3f86d41f353e7680b597512ffe7eac5d0"}, - {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:efec946f331349dfc4ae9d0e034c263ddde19414fe5128580f512619abed05f1"}, - {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b80b4690bbff51a034bfde9c9f6bf9357f0a8c61f548942b80f7b66356508bf5"}, - {file = "rpds_py-0.21.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:085ed25baac88953d4283e5b5bd094b155075bb40d07c29c4f073e10623f9f2e"}, - {file = "rpds_py-0.21.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:daa8efac2a1273eed2354397a51216ae1e198ecbce9036fba4e7610b308b6153"}, - {file = "rpds_py-0.21.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:95a5bad1ac8a5c77b4e658671642e4af3707f095d2b78a1fdd08af0dfb647624"}, - {file = "rpds_py-0.21.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3e53861b29a13d5b70116ea4230b5f0f3547b2c222c5daa090eb7c9c82d7f664"}, - {file = "rpds_py-0.21.0-cp310-none-win32.whl", hash = "sha256:ea3a6ac4d74820c98fcc9da4a57847ad2cc36475a8bd9683f32ab6d47a2bd682"}, - {file = "rpds_py-0.21.0-cp310-none-win_amd64.whl", hash = "sha256:b8f107395f2f1d151181880b69a2869c69e87ec079c49c0016ab96860b6acbe5"}, - {file = "rpds_py-0.21.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5555db3e618a77034954b9dc547eae94166391a98eb867905ec8fcbce1308d95"}, - {file = "rpds_py-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:97ef67d9bbc3e15584c2f3c74bcf064af36336c10d2e21a2131e123ce0f924c9"}, - {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab2c2a26d2f69cdf833174f4d9d86118edc781ad9a8fa13970b527bf8236027"}, - {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4e8921a259f54bfbc755c5bbd60c82bb2339ae0324163f32868f63f0ebb873d9"}, - {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a7ff941004d74d55a47f916afc38494bd1cfd4b53c482b77c03147c91ac0ac3"}, - {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5145282a7cd2ac16ea0dc46b82167754d5e103a05614b724457cffe614f25bd8"}, - {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de609a6f1b682f70bb7163da745ee815d8f230d97276db049ab447767466a09d"}, - {file = "rpds_py-0.21.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40c91c6e34cf016fa8e6b59d75e3dbe354830777fcfd74c58b279dceb7975b75"}, - {file = "rpds_py-0.21.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d2132377f9deef0c4db89e65e8bb28644ff75a18df5293e132a8d67748397b9f"}, - {file = "rpds_py-0.21.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0a9e0759e7be10109645a9fddaaad0619d58c9bf30a3f248a2ea57a7c417173a"}, - {file = "rpds_py-0.21.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9e20da3957bdf7824afdd4b6eeb29510e83e026473e04952dca565170cd1ecc8"}, - {file = "rpds_py-0.21.0-cp311-none-win32.whl", hash = "sha256:f71009b0d5e94c0e86533c0b27ed7cacc1239cb51c178fd239c3cfefefb0400a"}, - {file = "rpds_py-0.21.0-cp311-none-win_amd64.whl", hash = "sha256:e168afe6bf6ab7ab46c8c375606298784ecbe3ba31c0980b7dcbb9631dcba97e"}, - {file = "rpds_py-0.21.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:30b912c965b2aa76ba5168fd610087bad7fcde47f0a8367ee8f1876086ee6d1d"}, - {file = "rpds_py-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ca9989d5d9b1b300bc18e1801c67b9f6d2c66b8fd9621b36072ed1df2c977f72"}, - {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f54e7106f0001244a5f4cf810ba8d3f9c542e2730821b16e969d6887b664266"}, - {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fed5dfefdf384d6fe975cc026886aece4f292feaf69d0eeb716cfd3c5a4dd8be"}, - {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:590ef88db231c9c1eece44dcfefd7515d8bf0d986d64d0caf06a81998a9e8cab"}, - {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f983e4c2f603c95dde63df633eec42955508eefd8d0f0e6d236d31a044c882d7"}, - {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b229ce052ddf1a01c67d68166c19cb004fb3612424921b81c46e7ea7ccf7c3bf"}, - {file = "rpds_py-0.21.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ebf64e281a06c904a7636781d2e973d1f0926a5b8b480ac658dc0f556e7779f4"}, - {file = "rpds_py-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:998a8080c4495e4f72132f3d66ff91f5997d799e86cec6ee05342f8f3cda7dca"}, - {file = "rpds_py-0.21.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:98486337f7b4f3c324ab402e83453e25bb844f44418c066623db88e4c56b7c7b"}, - {file = "rpds_py-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a78d8b634c9df7f8d175451cfeac3810a702ccb85f98ec95797fa98b942cea11"}, - {file = "rpds_py-0.21.0-cp312-none-win32.whl", hash = "sha256:a58ce66847711c4aa2ecfcfaff04cb0327f907fead8945ffc47d9407f41ff952"}, - {file = "rpds_py-0.21.0-cp312-none-win_amd64.whl", hash = "sha256:e860f065cc4ea6f256d6f411aba4b1251255366e48e972f8a347cf88077b24fd"}, - {file = "rpds_py-0.21.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:ee4eafd77cc98d355a0d02f263efc0d3ae3ce4a7c24740010a8b4012bbb24937"}, - {file = "rpds_py-0.21.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:688c93b77e468d72579351a84b95f976bd7b3e84aa6686be6497045ba84be560"}, - {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c38dbf31c57032667dd5a2f0568ccde66e868e8f78d5a0d27dcc56d70f3fcd3b"}, - {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2d6129137f43f7fa02d41542ffff4871d4aefa724a5fe38e2c31a4e0fd343fb0"}, - {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:520ed8b99b0bf86a176271f6fe23024323862ac674b1ce5b02a72bfeff3fff44"}, - {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaeb25ccfb9b9014a10eaf70904ebf3f79faaa8e60e99e19eef9f478651b9b74"}, - {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af04ac89c738e0f0f1b913918024c3eab6e3ace989518ea838807177d38a2e94"}, - {file = "rpds_py-0.21.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9b76e2afd585803c53c5b29e992ecd183f68285b62fe2668383a18e74abe7a3"}, - {file = "rpds_py-0.21.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5afb5efde74c54724e1a01118c6e5c15e54e642c42a1ba588ab1f03544ac8c7a"}, - {file = "rpds_py-0.21.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:52c041802a6efa625ea18027a0723676a778869481d16803481ef6cc02ea8cb3"}, - {file = "rpds_py-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee1e4fc267b437bb89990b2f2abf6c25765b89b72dd4a11e21934df449e0c976"}, - {file = "rpds_py-0.21.0-cp313-none-win32.whl", hash = "sha256:0c025820b78817db6a76413fff6866790786c38f95ea3f3d3c93dbb73b632202"}, - {file = "rpds_py-0.21.0-cp313-none-win_amd64.whl", hash = "sha256:320c808df533695326610a1b6a0a6e98f033e49de55d7dc36a13c8a30cfa756e"}, - {file = "rpds_py-0.21.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:2c51d99c30091f72a3c5d126fad26236c3f75716b8b5e5cf8effb18889ced928"}, - {file = "rpds_py-0.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cbd7504a10b0955ea287114f003b7ad62330c9e65ba012c6223dba646f6ffd05"}, - {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6dcc4949be728ede49e6244eabd04064336012b37f5c2200e8ec8eb2988b209c"}, - {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f414da5c51bf350e4b7960644617c130140423882305f7574b6cf65a3081cecb"}, - {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9afe42102b40007f588666bc7de82451e10c6788f6f70984629db193849dced1"}, - {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b929c2bb6e29ab31f12a1117c39f7e6d6450419ab7464a4ea9b0b417174f044"}, - {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8404b3717da03cbf773a1d275d01fec84ea007754ed380f63dfc24fb76ce4592"}, - {file = "rpds_py-0.21.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e12bb09678f38b7597b8346983d2323a6482dcd59e423d9448108c1be37cac9d"}, - {file = "rpds_py-0.21.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:58a0e345be4b18e6b8501d3b0aa540dad90caeed814c515e5206bb2ec26736fd"}, - {file = "rpds_py-0.21.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c3761f62fcfccf0864cc4665b6e7c3f0c626f0380b41b8bd1ce322103fa3ef87"}, - {file = "rpds_py-0.21.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c2b2f71c6ad6c2e4fc9ed9401080badd1469fa9889657ec3abea42a3d6b2e1ed"}, - {file = "rpds_py-0.21.0-cp39-none-win32.whl", hash = "sha256:b21747f79f360e790525e6f6438c7569ddbfb1b3197b9e65043f25c3c9b489d8"}, - {file = "rpds_py-0.21.0-cp39-none-win_amd64.whl", hash = "sha256:0626238a43152918f9e72ede9a3b6ccc9e299adc8ade0d67c5e142d564c9a83d"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6b4ef7725386dc0762857097f6b7266a6cdd62bfd209664da6712cb26acef035"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:6bc0e697d4d79ab1aacbf20ee5f0df80359ecf55db33ff41481cf3e24f206919"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da52d62a96e61c1c444f3998c434e8b263c384f6d68aca8274d2e08d1906325c"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:98e4fe5db40db87ce1c65031463a760ec7906ab230ad2249b4572c2fc3ef1f9f"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30bdc973f10d28e0337f71d202ff29345320f8bc49a31c90e6c257e1ccef4333"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:faa5e8496c530f9c71f2b4e1c49758b06e5f4055e17144906245c99fa6d45356"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32eb88c30b6a4f0605508023b7141d043a79b14acb3b969aa0b4f99b25bc7d4a"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a89a8ce9e4e75aeb7fa5d8ad0f3fecdee813802592f4f46a15754dcb2fd6b061"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:241e6c125568493f553c3d0fdbb38c74babf54b45cef86439d4cd97ff8feb34d"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:3b766a9f57663396e4f34f5140b3595b233a7b146e94777b97a8413a1da1be18"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:af4a644bf890f56e41e74be7d34e9511e4954894d544ec6b8efe1e21a1a8da6c"}, - {file = "rpds_py-0.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3e30a69a706e8ea20444b98a49f386c17b26f860aa9245329bab0851ed100677"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:031819f906bb146561af051c7cef4ba2003d28cff07efacef59da973ff7969ba"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b876f2bc27ab5954e2fd88890c071bd0ed18b9c50f6ec3de3c50a5ece612f7a6"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc5695c321e518d9f03b7ea6abb5ea3af4567766f9852ad1560f501b17588c7b"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4de1da871b5c0fd5537b26a6fc6814c3cc05cabe0c941db6e9044ffbb12f04a"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:878f6fea96621fda5303a2867887686d7a198d9e0f8a40be100a63f5d60c88c9"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8eeec67590e94189f434c6d11c426892e396ae59e4801d17a93ac96b8c02a6c"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ff2eba7f6c0cb523d7e9cff0903f2fe1feff8f0b2ceb6bd71c0e20a4dcee271"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a429b99337062877d7875e4ff1a51fe788424d522bd64a8c0a20ef3021fdb6ed"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:d167e4dbbdac48bd58893c7e446684ad5d425b407f9336e04ab52e8b9194e2ed"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:4eb2de8a147ffe0626bfdc275fc6563aa7bf4b6db59cf0d44f0ccd6ca625a24e"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:e78868e98f34f34a88e23ee9ccaeeec460e4eaf6db16d51d7a9b883e5e785a5e"}, - {file = "rpds_py-0.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4991ca61656e3160cdaca4851151fd3f4a92e9eba5c7a530ab030d6aee96ec89"}, - {file = "rpds_py-0.21.0.tar.gz", hash = "sha256:ed6378c9d66d0de903763e7706383d60c33829581f0adff47b6535f1802fa6db"}, + {file = "rpds_py-0.22.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:6c7b99ca52c2c1752b544e310101b98a659b720b21db00e65edca34483259967"}, + {file = "rpds_py-0.22.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be2eb3f2495ba669d2a985f9b426c1797b7d48d6963899276d22f23e33d47e37"}, + {file = "rpds_py-0.22.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70eb60b3ae9245ddea20f8a4190bd79c705a22f8028aaf8bbdebe4716c3fab24"}, + {file = "rpds_py-0.22.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4041711832360a9b75cfb11b25a6a97c8fb49c07b8bd43d0d02b45d0b499a4ff"}, + {file = "rpds_py-0.22.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64607d4cbf1b7e3c3c8a14948b99345eda0e161b852e122c6bb71aab6d1d798c"}, + {file = "rpds_py-0.22.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e69b0a0e2537f26d73b4e43ad7bc8c8efb39621639b4434b76a3de50c6966e"}, + {file = "rpds_py-0.22.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc27863442d388870c1809a87507727b799c8460573cfbb6dc0eeaef5a11b5ec"}, + {file = "rpds_py-0.22.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e79dd39f1e8c3504be0607e5fc6e86bb60fe3584bec8b782578c3b0fde8d932c"}, + {file = "rpds_py-0.22.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e0fa2d4ec53dc51cf7d3bb22e0aa0143966119f42a0c3e4998293a3dd2856b09"}, + {file = "rpds_py-0.22.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00"}, + {file = "rpds_py-0.22.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cff63a0272fcd259dcc3be1657b07c929c466b067ceb1c20060e8d10af56f5bf"}, + {file = "rpds_py-0.22.3-cp310-cp310-win32.whl", hash = "sha256:9bd7228827ec7bb817089e2eb301d907c0d9827a9e558f22f762bb690b131652"}, + {file = "rpds_py-0.22.3-cp310-cp310-win_amd64.whl", hash = "sha256:9beeb01d8c190d7581a4d59522cd3d4b6887040dcfc744af99aa59fef3e041a8"}, + {file = "rpds_py-0.22.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d20cfb4e099748ea39e6f7b16c91ab057989712d31761d3300d43134e26e165f"}, + {file = "rpds_py-0.22.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:68049202f67380ff9aa52f12e92b1c30115f32e6895cd7198fa2a7961621fc5a"}, + {file = "rpds_py-0.22.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb4f868f712b2dd4bcc538b0a0c1f63a2b1d584c925e69a224d759e7070a12d5"}, + {file = "rpds_py-0.22.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bc51abd01f08117283c5ebf64844a35144a0843ff7b2983e0648e4d3d9f10dbb"}, + {file = "rpds_py-0.22.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0f3cec041684de9a4684b1572fe28c7267410e02450f4561700ca5a3bc6695a2"}, + {file = "rpds_py-0.22.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7ef9d9da710be50ff6809fed8f1963fecdfecc8b86656cadfca3bc24289414b0"}, + {file = "rpds_py-0.22.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59f4a79c19232a5774aee369a0c296712ad0e77f24e62cad53160312b1c1eaa1"}, + {file = "rpds_py-0.22.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a60bce91f81ddaac922a40bbb571a12c1070cb20ebd6d49c48e0b101d87300d"}, + {file = "rpds_py-0.22.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e89391e6d60251560f0a8f4bd32137b077a80d9b7dbe6d5cab1cd80d2746f648"}, + {file = "rpds_py-0.22.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e3fb866d9932a3d7d0c82da76d816996d1667c44891bd861a0f97ba27e84fc74"}, + {file = "rpds_py-0.22.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1352ae4f7c717ae8cba93421a63373e582d19d55d2ee2cbb184344c82d2ae55a"}, + {file = "rpds_py-0.22.3-cp311-cp311-win32.whl", hash = "sha256:b0b4136a252cadfa1adb705bb81524eee47d9f6aab4f2ee4fa1e9d3cd4581f64"}, + {file = "rpds_py-0.22.3-cp311-cp311-win_amd64.whl", hash = "sha256:8bd7c8cfc0b8247c8799080fbff54e0b9619e17cdfeb0478ba7295d43f635d7c"}, + {file = "rpds_py-0.22.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:27e98004595899949bd7a7b34e91fa7c44d7a97c40fcaf1d874168bb652ec67e"}, + {file = "rpds_py-0.22.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1978d0021e943aae58b9b0b196fb4895a25cc53d3956b8e35e0b7682eefb6d56"}, + {file = "rpds_py-0.22.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:655ca44a831ecb238d124e0402d98f6212ac527a0ba6c55ca26f616604e60a45"}, + {file = "rpds_py-0.22.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e"}, + {file = "rpds_py-0.22.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22bebe05a9ffc70ebfa127efbc429bc26ec9e9b4ee4d15a740033efda515cf3d"}, + {file = "rpds_py-0.22.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3af6e48651c4e0d2d166dc1b033b7042ea3f871504b6805ba5f4fe31581d8d38"}, + {file = "rpds_py-0.22.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e67ba3c290821343c192f7eae1d8fd5999ca2dc99994114643e2f2d3e6138b15"}, + {file = "rpds_py-0.22.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02fbb9c288ae08bcb34fb41d516d5eeb0455ac35b5512d03181d755d80810059"}, + {file = "rpds_py-0.22.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f56a6b404f74ab372da986d240e2e002769a7d7102cc73eb238a4f72eec5284e"}, + {file = "rpds_py-0.22.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0a0461200769ab3b9ab7e513f6013b7a97fdeee41c29b9db343f3c5a8e2b9e61"}, + {file = "rpds_py-0.22.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8633e471c6207a039eff6aa116e35f69f3156b3989ea3e2d755f7bc41754a4a7"}, + {file = "rpds_py-0.22.3-cp312-cp312-win32.whl", hash = "sha256:593eba61ba0c3baae5bc9be2f5232430453fb4432048de28399ca7376de9c627"}, + {file = "rpds_py-0.22.3-cp312-cp312-win_amd64.whl", hash = "sha256:d115bffdd417c6d806ea9069237a4ae02f513b778e3789a359bc5856e0404cc4"}, + {file = "rpds_py-0.22.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:ea7433ce7e4bfc3a85654aeb6747babe3f66eaf9a1d0c1e7a4435bbdf27fea84"}, + {file = "rpds_py-0.22.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6dd9412824c4ce1aca56c47b0991e65bebb7ac3f4edccfd3f156150c96a7bf25"}, + {file = "rpds_py-0.22.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20070c65396f7373f5df4005862fa162db5d25d56150bddd0b3e8214e8ef45b4"}, + {file = "rpds_py-0.22.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0b09865a9abc0ddff4e50b5ef65467cd94176bf1e0004184eb915cbc10fc05c5"}, + {file = "rpds_py-0.22.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3453e8d41fe5f17d1f8e9c383a7473cd46a63661628ec58e07777c2fff7196dc"}, + {file = "rpds_py-0.22.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5d36399a1b96e1a5fdc91e0522544580dbebeb1f77f27b2b0ab25559e103b8b"}, + {file = "rpds_py-0.22.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:009de23c9c9ee54bf11303a966edf4d9087cd43a6003672e6aa7def643d06518"}, + {file = "rpds_py-0.22.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1aef18820ef3e4587ebe8b3bc9ba6e55892a6d7b93bac6d29d9f631a3b4befbd"}, + {file = "rpds_py-0.22.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f60bd8423be1d9d833f230fdbccf8f57af322d96bcad6599e5a771b151398eb2"}, + {file = "rpds_py-0.22.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:62d9cfcf4948683a18a9aff0ab7e1474d407b7bab2ca03116109f8464698ab16"}, + {file = "rpds_py-0.22.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9253fc214112405f0afa7db88739294295f0e08466987f1d70e29930262b4c8f"}, + {file = "rpds_py-0.22.3-cp313-cp313-win32.whl", hash = "sha256:fb0ba113b4983beac1a2eb16faffd76cb41e176bf58c4afe3e14b9c681f702de"}, + {file = "rpds_py-0.22.3-cp313-cp313-win_amd64.whl", hash = "sha256:c58e2339def52ef6b71b8f36d13c3688ea23fa093353f3a4fee2556e62086ec9"}, + {file = "rpds_py-0.22.3-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:f82a116a1d03628a8ace4859556fb39fd1424c933341a08ea3ed6de1edb0283b"}, + {file = "rpds_py-0.22.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3dfcbc95bd7992b16f3f7ba05af8a64ca694331bd24f9157b49dadeeb287493b"}, + {file = "rpds_py-0.22.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59259dc58e57b10e7e18ce02c311804c10c5a793e6568f8af4dead03264584d1"}, + {file = "rpds_py-0.22.3-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5725dd9cc02068996d4438d397e255dcb1df776b7ceea3b9cb972bdb11260a83"}, + {file = "rpds_py-0.22.3-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99b37292234e61325e7a5bb9689e55e48c3f5f603af88b1642666277a81f1fbd"}, + {file = "rpds_py-0.22.3-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27b1d3b3915a99208fee9ab092b8184c420f2905b7d7feb4aeb5e4a9c509b8a1"}, + {file = "rpds_py-0.22.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f612463ac081803f243ff13cccc648578e2279295048f2a8d5eb430af2bae6e3"}, + {file = "rpds_py-0.22.3-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f73d3fef726b3243a811121de45193c0ca75f6407fe66f3f4e183c983573e130"}, + {file = "rpds_py-0.22.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:3f21f0495edea7fdbaaa87e633a8689cd285f8f4af5c869f27bc8074638ad69c"}, + {file = "rpds_py-0.22.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:1e9663daaf7a63ceccbbb8e3808fe90415b0757e2abddbfc2e06c857bf8c5e2b"}, + {file = "rpds_py-0.22.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a76e42402542b1fae59798fab64432b2d015ab9d0c8c47ba7addddbaf7952333"}, + {file = "rpds_py-0.22.3-cp313-cp313t-win32.whl", hash = "sha256:69803198097467ee7282750acb507fba35ca22cc3b85f16cf45fb01cb9097730"}, + {file = "rpds_py-0.22.3-cp313-cp313t-win_amd64.whl", hash = "sha256:f5cf2a0c2bdadf3791b5c205d55a37a54025c6e18a71c71f82bb536cf9a454bf"}, + {file = "rpds_py-0.22.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:378753b4a4de2a7b34063d6f95ae81bfa7b15f2c1a04a9518e8644e81807ebea"}, + {file = "rpds_py-0.22.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3445e07bf2e8ecfeef6ef67ac83de670358abf2996916039b16a218e3d95e97e"}, + {file = "rpds_py-0.22.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b2513ba235829860b13faa931f3b6846548021846ac808455301c23a101689d"}, + {file = "rpds_py-0.22.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eaf16ae9ae519a0e237a0f528fd9f0197b9bb70f40263ee57ae53c2b8d48aeb3"}, + {file = "rpds_py-0.22.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:583f6a1993ca3369e0f80ba99d796d8e6b1a3a2a442dd4e1a79e652116413091"}, + {file = "rpds_py-0.22.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4617e1915a539a0d9a9567795023de41a87106522ff83fbfaf1f6baf8e85437e"}, + {file = "rpds_py-0.22.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c150c7a61ed4a4f4955a96626574e9baf1adf772c2fb61ef6a5027e52803543"}, + {file = "rpds_py-0.22.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fa4331c200c2521512595253f5bb70858b90f750d39b8cbfd67465f8d1b596d"}, + {file = "rpds_py-0.22.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:214b7a953d73b5e87f0ebece4a32a5bd83c60a3ecc9d4ec8f1dca968a2d91e99"}, + {file = "rpds_py-0.22.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f47ad3d5f3258bd7058d2d506852217865afefe6153a36eb4b6928758041d831"}, + {file = "rpds_py-0.22.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:f276b245347e6e36526cbd4a266a417796fc531ddf391e43574cf6466c492520"}, + {file = "rpds_py-0.22.3-cp39-cp39-win32.whl", hash = "sha256:bbb232860e3d03d544bc03ac57855cd82ddf19c7a07651a7c0fdb95e9efea8b9"}, + {file = "rpds_py-0.22.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfbc454a2880389dbb9b5b398e50d439e2e58669160f27b60e5eca11f68ae17c"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d48424e39c2611ee1b84ad0f44fb3b2b53d473e65de061e3f460fc0be5f1939d"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:24e8abb5878e250f2eb0d7859a8e561846f98910326d06c0d51381fed59357bd"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b232061ca880db21fa14defe219840ad9b74b6158adb52ddf0e87bead9e8493"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac0a03221cdb5058ce0167ecc92a8c89e8d0decdc9e99a2ec23380793c4dcb96"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb0c341fa71df5a4595f9501df4ac5abfb5a09580081dffbd1ddd4654e6e9123"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf9db5488121b596dbfc6718c76092fda77b703c1f7533a226a5a9f65248f8ad"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b8db6b5b2d4491ad5b6bdc2bc7c017eec108acbf4e6785f42a9eb0ba234f4c9"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b3d504047aba448d70cf6fa22e06cb09f7cbd761939fdd47604f5e007675c24e"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:e61b02c3f7a1e0b75e20c3978f7135fd13cb6cf551bf4a6d29b999a88830a338"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:e35ba67d65d49080e8e5a1dd40101fccdd9798adb9b050ff670b7d74fa41c566"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:26fd7cac7dd51011a245f29a2cc6489c4608b5a8ce8d75661bb4a1066c52dfbe"}, + {file = "rpds_py-0.22.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:177c7c0fce2855833819c98e43c262007f42ce86651ffbb84f37883308cb0e7d"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bb47271f60660803ad11f4c61b42242b8c1312a31c98c578f79ef9387bbde21c"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:70fb28128acbfd264eda9bf47015537ba3fe86e40d046eb2963d75024be4d055"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44d61b4b7d0c2c9ac019c314e52d7cbda0ae31078aabd0f22e583af3e0d79723"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f0e260eaf54380380ac3808aa4ebe2d8ca28b9087cf411649f96bad6900c728"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b25bc607423935079e05619d7de556c91fb6adeae9d5f80868dde3468657994b"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fb6116dfb8d1925cbdb52595560584db42a7f664617a1f7d7f6e32f138cdf37d"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a63cbdd98acef6570c62b92a1e43266f9e8b21e699c363c0fef13bd530799c11"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2b8f60e1b739a74bab7e01fcbe3dddd4657ec685caa04681df9d562ef15b625f"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2e8b55d8517a2fda8d95cb45d62a5a8bbf9dd0ad39c5b25c8833efea07b880ca"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:2de29005e11637e7a2361fa151f780ff8eb2543a0da1413bb951e9f14b699ef3"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:666ecce376999bf619756a24ce15bb14c5bfaf04bf00abc7e663ce17c3f34fe7"}, + {file = "rpds_py-0.22.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:5246b14ca64a8675e0a7161f7af68fe3e910e6b90542b4bfb5439ba752191df6"}, + {file = "rpds_py-0.22.3.tar.gz", hash = "sha256:e32fee8ab45d3c2db6da19a5323bc3362237c8b653c70194414b892fd06a080d"}, ] [[package]] name = "six" -version = "1.16.0" +version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] [[package]] @@ -2501,123 +2503,26 @@ testing = ["mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ru [[package]] name = "tokenizers" -version = "0.20.3" +version = "0.21.0" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "tokenizers-0.20.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:31ccab28dbb1a9fe539787210b0026e22debeab1662970f61c2d921f7557f7e4"}, - {file = "tokenizers-0.20.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6361191f762bda98c773da418cf511cbaa0cb8d0a1196f16f8c0119bde68ff8"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f128d5da1202b78fa0a10d8d938610472487da01b57098d48f7e944384362514"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79c4121a2e9433ad7ef0769b9ca1f7dd7fa4c0cd501763d0a030afcbc6384481"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7850fde24197fe5cd6556e2fdba53a6d3bae67c531ea33a3d7c420b90904141"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b357970c095dc134978a68c67d845a1e3803ab7c4fbb39195bde914e7e13cf8b"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a333d878c4970b72d6c07848b90c05f6b045cf9273fc2bc04a27211721ad6118"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd9fee817f655a8f50049f685e224828abfadd436b8ff67979fc1d054b435f1"}, - {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9e7816808b402129393a435ea2a509679b41246175d6e5e9f25b8692bfaa272b"}, - {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba96367db9d8a730d3a1d5996b4b7babb846c3994b8ef14008cd8660f55db59d"}, - {file = "tokenizers-0.20.3-cp310-none-win32.whl", hash = "sha256:ee31ba9d7df6a98619426283e80c6359f167e2e9882d9ce1b0254937dbd32f3f"}, - {file = "tokenizers-0.20.3-cp310-none-win_amd64.whl", hash = "sha256:a845c08fdad554fe0871d1255df85772f91236e5fd6b9287ef8b64f5807dbd0c"}, - {file = "tokenizers-0.20.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:585b51e06ca1f4839ce7759941e66766d7b060dccfdc57c4ca1e5b9a33013a90"}, - {file = "tokenizers-0.20.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61cbf11954f3b481d08723ebd048ba4b11e582986f9be74d2c3bdd9293a4538d"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef820880d5e4e8484e2fa54ff8d297bb32519eaa7815694dc835ace9130a3eea"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:67ef4dcb8841a4988cd00dd288fb95dfc8e22ed021f01f37348fd51c2b055ba9"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff1ef8bd47a02b0dc191688ccb4da53600df5d4c9a05a4b68e1e3de4823e78eb"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:444d188186eab3148baf0615b522461b41b1f0cd58cd57b862ec94b6ac9780f1"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37c04c032c1442740b2c2d925f1857885c07619224a533123ac7ea71ca5713da"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:453c7769d22231960ee0e883d1005c93c68015025a5e4ae56275406d94a3c907"}, - {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4bb31f7b2847e439766aaa9cc7bccf7ac7088052deccdb2275c952d96f691c6a"}, - {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:843729bf0f991b29655a069a2ff58a4c24375a553c70955e15e37a90dd4e045c"}, - {file = "tokenizers-0.20.3-cp311-none-win32.whl", hash = "sha256:efcce3a927b1e20ca694ba13f7a68c59b0bd859ef71e441db68ee42cf20c2442"}, - {file = "tokenizers-0.20.3-cp311-none-win_amd64.whl", hash = "sha256:88301aa0801f225725b6df5dea3d77c80365ff2362ca7e252583f2b4809c4cc0"}, - {file = "tokenizers-0.20.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:49d12a32e190fad0e79e5bdb788d05da2f20d8e006b13a70859ac47fecf6ab2f"}, - {file = "tokenizers-0.20.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:282848cacfb9c06d5e51489f38ec5aa0b3cd1e247a023061945f71f41d949d73"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abe4e08c7d0cd6154c795deb5bf81d2122f36daf075e0c12a8b050d824ef0a64"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca94fc1b73b3883c98f0c88c77700b13d55b49f1071dfd57df2b06f3ff7afd64"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef279c7e239f95c8bdd6ff319d9870f30f0d24915b04895f55b1adcf96d6c60d"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16384073973f6ccbde9852157a4fdfe632bb65208139c9d0c0bd0176a71fd67f"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:312d522caeb8a1a42ebdec87118d99b22667782b67898a76c963c058a7e41d4f"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2b7cb962564785a83dafbba0144ecb7f579f1d57d8c406cdaa7f32fe32f18ad"}, - {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:124c5882ebb88dadae1fc788a582299fcd3a8bd84fc3e260b9918cf28b8751f5"}, - {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2b6e54e71f84c4202111a489879005cb14b92616a87417f6c102c833af961ea2"}, - {file = "tokenizers-0.20.3-cp312-none-win32.whl", hash = "sha256:83d9bfbe9af86f2d9df4833c22e94d94750f1d0cd9bfb22a7bb90a86f61cdb1c"}, - {file = "tokenizers-0.20.3-cp312-none-win_amd64.whl", hash = "sha256:44def74cee574d609a36e17c8914311d1b5dbcfe37c55fd29369d42591b91cf2"}, - {file = "tokenizers-0.20.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e0b630e0b536ef0e3c8b42c685c1bc93bd19e98c0f1543db52911f8ede42cf84"}, - {file = "tokenizers-0.20.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a02d160d2b19bcbfdf28bd9a4bf11be4cb97d0499c000d95d4c4b1a4312740b6"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e3d80d89b068bc30034034b5319218c7c0a91b00af19679833f55f3becb6945"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:174a54910bed1b089226512b4458ea60d6d6fd93060254734d3bc3540953c51c"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:098b8a632b8656aa5802c46689462c5c48f02510f24029d71c208ec2c822e771"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:78c8c143e3ae41e718588281eb3e212c2b31623c9d6d40410ec464d7d6221fb5"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b26b0aadb18cd8701077362ba359a06683662d5cafe3e8e8aba10eb05c037f1"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07d7851a72717321022f3774e84aa9d595a041d643fafa2e87fbc9b18711dac0"}, - {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:bd44e48a430ada902c6266a8245f5036c4fe744fcb51f699999fbe82aa438797"}, - {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a4c186bb006ccbe1f5cc4e0380d1ce7806f5955c244074fd96abc55e27b77f01"}, - {file = "tokenizers-0.20.3-cp313-none-win32.whl", hash = "sha256:6e19e0f1d854d6ab7ea0c743d06e764d1d9a546932be0a67f33087645f00fe13"}, - {file = "tokenizers-0.20.3-cp313-none-win_amd64.whl", hash = "sha256:d50ede425c7e60966a9680d41b58b3a0950afa1bb570488e2972fa61662c4273"}, - {file = "tokenizers-0.20.3-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:9adda1ff5fb9dcdf899ceca672a4e2ce9e797adb512a6467305ca3d8bfcfbdd0"}, - {file = "tokenizers-0.20.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:6dde2cae6004ba7a3badff4a11911cae03ebf23e97eebfc0e71fef2530e5074f"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4a7fd678b35614fca708579eb95b7587a5e8a6d328171bd2488fd9f27d82be4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b80e3c7283a01a356bd2210f53d1a4a5d32b269c2024389ed0173137708d50e"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8cc0e8176b762973758a77f0d9c4467d310e33165fb74173418ca3734944da4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5634b2e2f5f3d2b4439d2d74066e22eb4b1f04f3fea05cb2a3c12d89b5a3bcd"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b4ba635165bc1ea46f2da8e5d80b5f70f6ec42161e38d96dbef33bb39df73964"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18e4c7c64172e7789bd8b07aa3087ea87c4c4de7e90937a2aa036b5d92332536"}, - {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1f74909ef7675c26d4095a817ec3393d67f3158ca4836c233212e5613ef640c4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0e9b81321a1e05b16487d312b4264984513f8b4a7556229cafac6e88c2036b09"}, - {file = "tokenizers-0.20.3-cp37-none-win32.whl", hash = "sha256:ab48184cd58b4a03022a2ec75b54c9f600ffea9a733612c02325ed636f353729"}, - {file = "tokenizers-0.20.3-cp37-none-win_amd64.whl", hash = "sha256:60ac483cebee1c12c71878523e768df02fa17e4c54412966cb3ac862c91b36c1"}, - {file = "tokenizers-0.20.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3229ef103c89583d10b9378afa5d601b91e6337530a0988e17ca8d635329a996"}, - {file = "tokenizers-0.20.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6ac52cc24bad3de865c7e65b1c4e7b70d00938a8ae09a92a453b8f676e714ad5"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04627b7b502fa6a2a005e1bd446fa4247d89abcb1afaa1b81eb90e21aba9a60f"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c27ceb887f0e81a3c377eb4605dca7a95a81262761c0fba308d627b2abb98f2b"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65ab780194da4e1fcf5670523a2f377c4838ebf5249efe41fa1eddd2a84fb49d"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98d343134f47159e81f7f242264b0eb222e6b802f37173c8d7d7b64d5c9d1388"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2475bb004ab2009d29aff13b5047bfdb3d4b474f0aa9d4faa13a7f34dbbbb43"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b6583a65c01db1197c1eb36857ceba8ec329d53afadd268b42a6b04f4965724"}, - {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:62d00ba208358c037eeab7bfc00a905adc67b2d31b68ab40ed09d75881e114ea"}, - {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0fc7a39e5bedc817bda395a798dfe2d9c5f7c71153c90d381b5135a0328d9520"}, - {file = "tokenizers-0.20.3-cp38-none-win32.whl", hash = "sha256:84d40ee0f8550d64d3ea92dd7d24a8557a9172165bdb986c9fb2503b4fe4e3b6"}, - {file = "tokenizers-0.20.3-cp38-none-win_amd64.whl", hash = "sha256:205a45246ed7f1718cf3785cff88450ba603352412aaf220ace026384aa3f1c0"}, - {file = "tokenizers-0.20.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:93e37f0269a11dc3b1a953f1fca9707f0929ebf8b4063c591c71a0664219988e"}, - {file = "tokenizers-0.20.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f4cb0c614b0135e781de96c2af87e73da0389ac1458e2a97562ed26e29490d8d"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7eb2fb1c432f5746b22f8a7f09fc18c4156cb0031c77f53cb19379d82d43297a"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfa8d029bb156181b006643309d6b673615a24e4ed24cf03aa191d599b996f51"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f90549622de3bf476ad9f1dd6f3f952ec3ed6ab8615ae88ef060d0c5bfad55d"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1d469c74eebf5c43fd61cd9b030e271d17198edd7bd45392e03a3c091d7d6d4"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bee8f53b2594749f4460d53253bae55d718f04e9b633efa0f5df8938bd98e4f0"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:938441babf3e5720e4459e306ef2809fb267680df9d1ff2873458b22aef60248"}, - {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7310ab23d7b0caebecc0e8be11a1146f320f5f07284000f6ea54793e83de1b75"}, - {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:16121eb030a2b13094cfec936b0c12e8b4063c5f839591ea7d0212336d8f9921"}, - {file = "tokenizers-0.20.3-cp39-none-win32.whl", hash = "sha256:401cc21ef642ee235985d747f65e18f639464d377c70836c9003df208d582064"}, - {file = "tokenizers-0.20.3-cp39-none-win_amd64.whl", hash = "sha256:7498f3ea7746133335a6adb67a77cf77227a8b82c8483f644a2e5f86fea42b8d"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e919f2e3e68bb51dc31de4fcbbeff3bdf9c1cad489044c75e2b982a91059bd3c"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b8e9608f2773996cc272156e305bd79066163a66b0390fe21750aff62df1ac07"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39270a7050deaf50f7caff4c532c01b3c48f6608d42b3eacdebdc6795478c8df"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e005466632b1c5d2d2120f6de8aa768cc9d36cd1ab7d51d0c27a114c91a1e6ee"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a07962340b36189b6c8feda552ea1bfeee6cf067ff922a1d7760662c2ee229e5"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:55046ad3dd5f2b3c67501fcc8c9cbe3e901d8355f08a3b745e9b57894855f85b"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:efcf0eb939988b627558aaf2b9dc3e56d759cad2e0cfa04fcab378e4b48fc4fd"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f3558a7ae6a6d38a77dfce12172a1e2e1bf3e8871e744a1861cd7591ea9ebe24"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d53029fe44bc70c3ff14ef512460a0cf583495a0f8e2f4b70e26eb9438e38a9"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a2a56397b2bec5a629b516b23f0f8a3e4f978c7488d4a299980f8375954b85"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e5bfaae740ef9ece000f8a07e78ac0e2b085c5ce9648f8593ddf0243c9f76d"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fbaf3ea28fedfb2283da60e710aff25492e795a7397cad8a50f1e079b65a5a70"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c47c037116310dc976eb96b008e41b9cfaba002ed8005848d4d632ee0b7ba9ae"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c31751f0721f58f5e19bb27c1acc259aeff860d8629c4e1a900b26a1979ada8e"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:c697cbd3be7a79ea250ea5f380d6f12e534c543cfb137d5c734966b3ee4f34cc"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48971b88ef9130bf35b41b35fd857c3c4dae4a9cd7990ebc7fc03e59cc92438"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e615de179bbe060ab33773f0d98a8a8572b5883dd7dac66c1de8c056c7e748c"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da1ec842035ed9999c62e45fbe0ff14b7e8a7e02bb97688cc6313cf65e5cd755"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6ee4954c1dd23aadc27958dad759006e71659d497dcb0ef0c7c87ea992c16ebd"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3eda46ca402751ec82553a321bf35a617b76bbed7586e768c02ccacbdda94d6d"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:de082392a85eb0055cc055c535bff2f0cc15d7a000bdc36fbf601a0f3cf8507a"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c3db46cc0647bfd88263afdb739b92017a02a87ee30945cb3e86c7e25c7c9917"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a292392f24ab9abac5cfa8197e5a6208f2e43723420217e1ceba0b4ec77816ac"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dcd91f4e60f62b20d83a87a84fe062035a1e3ff49a8c2bbdeb2d441c8e311f4"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900991a2b8ee35961b1095db7e265342e0e42a84c1a594823d5ee9f8fb791958"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5a8d8261ca2133d4f98aa9627c748189502b3787537ba3d7e2beb4f7cfc5d627"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c4fd4d71e6deb6ddf99d8d0eab87d1d16f635898906e631914a9bae8ae9f2cfb"}, - {file = "tokenizers-0.20.3.tar.gz", hash = "sha256:2278b34c5d0dd78e087e1ca7f9b1dcbf129d80211afa645f214bd6e051037539"}, + {file = "tokenizers-0.21.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3c4c93eae637e7d2aaae3d376f06085164e1660f89304c0ab2b1d08a406636b2"}, + {file = "tokenizers-0.21.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:f53ea537c925422a2e0e92a24cce96f6bc5046bbef24a1652a5edc8ba975f62e"}, + {file = "tokenizers-0.21.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b177fb54c4702ef611de0c069d9169f0004233890e0c4c5bd5508ae05abf193"}, + {file = "tokenizers-0.21.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6b43779a269f4629bebb114e19c3fca0223296ae9fea8bb9a7a6c6fb0657ff8e"}, + {file = "tokenizers-0.21.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9aeb255802be90acfd363626753fda0064a8df06031012fe7d52fd9a905eb00e"}, + {file = "tokenizers-0.21.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8b09dbeb7a8d73ee204a70f94fc06ea0f17dcf0844f16102b9f414f0b7463ba"}, + {file = "tokenizers-0.21.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:400832c0904f77ce87c40f1a8a27493071282f785724ae62144324f171377273"}, + {file = "tokenizers-0.21.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84ca973b3a96894d1707e189c14a774b701596d579ffc7e69debfc036a61a04"}, + {file = "tokenizers-0.21.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:eb7202d231b273c34ec67767378cd04c767e967fda12d4a9e36208a34e2f137e"}, + {file = "tokenizers-0.21.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:089d56db6782a73a27fd8abf3ba21779f5b85d4a9f35e3b493c7bbcbbf0d539b"}, + {file = "tokenizers-0.21.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:c87ca3dc48b9b1222d984b6b7490355a6fdb411a2d810f6f05977258400ddb74"}, + {file = "tokenizers-0.21.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4145505a973116f91bc3ac45988a92e618a6f83eb458f49ea0790df94ee243ff"}, + {file = "tokenizers-0.21.0-cp39-abi3-win32.whl", hash = "sha256:eb1702c2f27d25d9dd5b389cc1f2f51813e99f8ca30d9e25348db6585a97e24a"}, + {file = "tokenizers-0.21.0-cp39-abi3-win_amd64.whl", hash = "sha256:87841da5a25a3a5f70c102de371db120f41873b854ba65e52bccd57df5a3780c"}, + {file = "tokenizers-0.21.0.tar.gz", hash = "sha256:ee0894bf311b75b0c03079f33859ae4b2334d675d4e93f5a4132e1eae2834fe4"}, ] [package.dependencies] @@ -2630,13 +2535,43 @@ testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"] [[package]] name = "tomli" -version = "2.0.2" +version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, - {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] [[package]] @@ -2652,20 +2587,20 @@ files = [ [[package]] name = "tqdm" -version = "4.67.0" +version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.67.0-py3-none-any.whl", hash = "sha256:0cd8af9d56911acab92182e88d763100d4788bdf421d251616040cc4d44863be"}, - {file = "tqdm-4.67.0.tar.gz", hash = "sha256:fe5a6f95e6fe0b9755e9469b77b9c3cf850048224ecaa8293d7d2d31f97d869a"}, + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] discord = ["requests"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] @@ -2684,13 +2619,13 @@ files = [ [[package]] name = "urllib3" -version = "2.2.3" +version = "2.3.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, - {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, + {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, + {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, ] [package.extras] @@ -2701,13 +2636,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "win32-setctime" -version = "1.1.0" +version = "1.2.0" description = "A small Python utility to set file creation time on Windows" optional = false python-versions = ">=3.5" files = [ - {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, - {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, + {file = "win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390"}, + {file = "win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0"}, ] [package.extras] @@ -2715,93 +2650,93 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [[package]] name = "yarl" -version = "1.17.1" +version = "1.18.3" description = "Yet another URL library" optional = false python-versions = ">=3.9" files = [ - {file = "yarl-1.17.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1794853124e2f663f0ea54efb0340b457f08d40a1cef78edfa086576179c91"}, - {file = "yarl-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fbea1751729afe607d84acfd01efd95e3b31db148a181a441984ce9b3d3469da"}, - {file = "yarl-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ee427208c675f1b6e344a1f89376a9613fc30b52646a04ac0c1f6587c7e46ec"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b74ff4767d3ef47ffe0cd1d89379dc4d828d4873e5528976ced3b44fe5b0a21"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62a91aefff3d11bf60e5956d340eb507a983a7ec802b19072bb989ce120cd948"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:846dd2e1243407133d3195d2d7e4ceefcaa5f5bf7278f0a9bda00967e6326b04"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e844be8d536afa129366d9af76ed7cb8dfefec99f5f1c9e4f8ae542279a6dc3"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc7c92c1baa629cb03ecb0c3d12564f172218fb1739f54bf5f3881844daadc6d"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae3476e934b9d714aa8000d2e4c01eb2590eee10b9d8cd03e7983ad65dfbfcba"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c7e177c619342e407415d4f35dec63d2d134d951e24b5166afcdfd1362828e17"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64cc6e97f14cf8a275d79c5002281f3040c12e2e4220623b5759ea7f9868d6a5"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:84c063af19ef5130084db70ada40ce63a84f6c1ef4d3dbc34e5e8c4febb20822"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:482c122b72e3c5ec98f11457aeb436ae4aecca75de19b3d1de7cf88bc40db82f"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:380e6c38ef692b8fd5a0f6d1fa8774d81ebc08cfbd624b1bca62a4d4af2f9931"}, - {file = "yarl-1.17.1-cp310-cp310-win32.whl", hash = "sha256:16bca6678a83657dd48df84b51bd56a6c6bd401853aef6d09dc2506a78484c7b"}, - {file = "yarl-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:561c87fea99545ef7d692403c110b2f99dced6dff93056d6e04384ad3bc46243"}, - {file = "yarl-1.17.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cbad927ea8ed814622305d842c93412cb47bd39a496ed0f96bfd42b922b4a217"}, - {file = "yarl-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fca4b4307ebe9c3ec77a084da3a9d1999d164693d16492ca2b64594340999988"}, - {file = "yarl-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff5c6771c7e3511a06555afa317879b7db8d640137ba55d6ab0d0c50425cab75"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b29beab10211a746f9846baa39275e80034e065460d99eb51e45c9a9495bcca"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a52a1ffdd824fb1835272e125385c32fd8b17fbdefeedcb4d543cc23b332d74"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58c8e9620eb82a189c6c40cb6b59b4e35b2ee68b1f2afa6597732a2b467d7e8f"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d216e5d9b8749563c7f2c6f7a0831057ec844c68b4c11cb10fc62d4fd373c26d"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881764d610e3269964fc4bb3c19bb6fce55422828e152b885609ec176b41cf11"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8c79e9d7e3d8a32d4824250a9c6401194fb4c2ad9a0cec8f6a96e09a582c2cc0"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:299f11b44d8d3a588234adbe01112126010bd96d9139c3ba7b3badd9829261c3"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:cc7d768260f4ba4ea01741c1b5fe3d3a6c70eb91c87f4c8761bbcce5181beafe"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:de599af166970d6a61accde358ec9ded821234cbbc8c6413acfec06056b8e860"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2b24ec55fad43e476905eceaf14f41f6478780b870eda5d08b4d6de9a60b65b4"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9fb815155aac6bfa8d86184079652c9715c812d506b22cfa369196ef4e99d1b4"}, - {file = "yarl-1.17.1-cp311-cp311-win32.whl", hash = "sha256:7615058aabad54416ddac99ade09a5510cf77039a3b903e94e8922f25ed203d7"}, - {file = "yarl-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:14bc88baa44e1f84164a392827b5defb4fa8e56b93fecac3d15315e7c8e5d8b3"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:327828786da2006085a4d1feb2594de6f6d26f8af48b81eb1ae950c788d97f61"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc353841428d56b683a123a813e6a686e07026d6b1c5757970a877195f880c2d"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c73df5b6e8fabe2ddb74876fb82d9dd44cbace0ca12e8861ce9155ad3c886139"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdff5e0995522706c53078f531fb586f56de9c4c81c243865dd5c66c132c3b5"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06157fb3c58f2736a5e47c8fcbe1afc8b5de6fb28b14d25574af9e62150fcaac"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1654ec814b18be1af2c857aa9000de7a601400bd4c9ca24629b18486c2e35463"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6595c852ca544aaeeb32d357e62c9c780eac69dcd34e40cae7b55bc4fb1147"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459e81c2fb920b5f5df744262d1498ec2c8081acdcfe18181da44c50f51312f7"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e48cdb8226644e2fbd0bdb0a0f87906a3db07087f4de77a1b1b1ccfd9e93685"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d9b6b28a57feb51605d6ae5e61a9044a31742db557a3b851a74c13bc61de5172"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e594b22688d5747b06e957f1ef822060cb5cb35b493066e33ceac0cf882188b7"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5f236cb5999ccd23a0ab1bd219cfe0ee3e1c1b65aaf6dd3320e972f7ec3a39da"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a2a64e62c7a0edd07c1c917b0586655f3362d2c2d37d474db1a509efb96fea1c"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d0eea830b591dbc68e030c86a9569826145df485b2b4554874b07fea1275a199"}, - {file = "yarl-1.17.1-cp312-cp312-win32.whl", hash = "sha256:46ddf6e0b975cd680eb83318aa1d321cb2bf8d288d50f1754526230fcf59ba96"}, - {file = "yarl-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:117ed8b3732528a1e41af3aa6d4e08483c2f0f2e3d3d7dca7cf538b3516d93df"}, - {file = "yarl-1.17.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5d1d42556b063d579cae59e37a38c61f4402b47d70c29f0ef15cee1acaa64488"}, - {file = "yarl-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0167540094838ee9093ef6cc2c69d0074bbf84a432b4995835e8e5a0d984374"}, - {file = "yarl-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2f0a6423295a0d282d00e8701fe763eeefba8037e984ad5de44aa349002562ac"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5b078134f48552c4d9527db2f7da0b5359abd49393cdf9794017baec7506170"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d401f07261dc5aa36c2e4efc308548f6ae943bfff20fcadb0a07517a26b196d8"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5f1ac7359e17efe0b6e5fec21de34145caef22b260e978336f325d5c84e6938"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f63d176a81555984e91f2c84c2a574a61cab7111cc907e176f0f01538e9ff6e"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e275792097c9f7e80741c36de3b61917aebecc08a67ae62899b074566ff8556"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:81713b70bea5c1386dc2f32a8f0dab4148a2928c7495c808c541ee0aae614d67"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:aa46dce75078fceaf7cecac5817422febb4355fbdda440db55206e3bd288cfb8"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1ce36ded585f45b1e9bb36d0ae94765c6608b43bd2e7f5f88079f7a85c61a4d3"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2d374d70fdc36f5863b84e54775452f68639bc862918602d028f89310a034ab0"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2d9f0606baaec5dd54cb99667fcf85183a7477f3766fbddbe3f385e7fc253299"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b0341e6d9a0c0e3cdc65857ef518bb05b410dbd70d749a0d33ac0f39e81a4258"}, - {file = "yarl-1.17.1-cp313-cp313-win32.whl", hash = "sha256:2e7ba4c9377e48fb7b20dedbd473cbcbc13e72e1826917c185157a137dac9df2"}, - {file = "yarl-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:949681f68e0e3c25377462be4b658500e85ca24323d9619fdc41f68d46a1ffda"}, - {file = "yarl-1.17.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8994b29c462de9a8fce2d591028b986dbbe1b32f3ad600b2d3e1c482c93abad6"}, - {file = "yarl-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f9cbfbc5faca235fbdf531b93aa0f9f005ec7d267d9d738761a4d42b744ea159"}, - {file = "yarl-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b40d1bf6e6f74f7c0a567a9e5e778bbd4699d1d3d2c0fe46f4b717eef9e96b95"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5efe0661b9fcd6246f27957f6ae1c0eb29bc60552820f01e970b4996e016004"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5c4804e4039f487e942c13381e6c27b4b4e66066d94ef1fae3f6ba8b953f383"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5d6a6c9602fd4598fa07e0389e19fe199ae96449008d8304bf5d47cb745462e"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f4c9156c4d1eb490fe374fb294deeb7bc7eaccda50e23775b2354b6a6739934"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6324274b4e0e2fa1b3eccb25997b1c9ed134ff61d296448ab8269f5ac068c4c"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d8a8b74d843c2638f3864a17d97a4acda58e40d3e44b6303b8cc3d3c44ae2d29"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:7fac95714b09da9278a0b52e492466f773cfe37651cf467a83a1b659be24bf71"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c180ac742a083e109c1a18151f4dd8675f32679985a1c750d2ff806796165b55"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:578d00c9b7fccfa1745a44f4eddfdc99d723d157dad26764538fbdda37209857"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1a3b91c44efa29e6c8ef8a9a2b583347998e2ba52c5d8280dbd5919c02dfc3b5"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a7ac5b4984c468ce4f4a553df281450df0a34aefae02e58d77a0847be8d1e11f"}, - {file = "yarl-1.17.1-cp39-cp39-win32.whl", hash = "sha256:7294e38f9aa2e9f05f765b28ffdc5d81378508ce6dadbe93f6d464a8c9594473"}, - {file = "yarl-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:eb6dce402734575e1a8cc0bb1509afca508a400a57ce13d306ea2c663bad1138"}, - {file = "yarl-1.17.1-py3-none-any.whl", hash = "sha256:f1790a4b1e8e8e028c391175433b9c8122c39b46e1663228158e61e6f915bf06"}, - {file = "yarl-1.17.1.tar.gz", hash = "sha256:067a63fcfda82da6b198fa73079b1ca40b7c9b7994995b6ee38acda728b64d47"}, + {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34"}, + {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7"}, + {file = "yarl-1.18.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690"}, + {file = "yarl-1.18.3-cp310-cp310-win32.whl", hash = "sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6"}, + {file = "yarl-1.18.3-cp310-cp310-win_amd64.whl", hash = "sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a"}, + {file = "yarl-1.18.3-cp311-cp311-win32.whl", hash = "sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1"}, + {file = "yarl-1.18.3-cp311-cp311-win_amd64.whl", hash = "sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285"}, + {file = "yarl-1.18.3-cp312-cp312-win32.whl", hash = "sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2"}, + {file = "yarl-1.18.3-cp312-cp312-win_amd64.whl", hash = "sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8"}, + {file = "yarl-1.18.3-cp313-cp313-win32.whl", hash = "sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d"}, + {file = "yarl-1.18.3-cp313-cp313-win_amd64.whl", hash = "sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1"}, + {file = "yarl-1.18.3-cp39-cp39-win32.whl", hash = "sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5"}, + {file = "yarl-1.18.3-cp39-cp39-win_amd64.whl", hash = "sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9"}, + {file = "yarl-1.18.3-py3-none-any.whl", hash = "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b"}, + {file = "yarl-1.18.3.tar.gz", hash = "sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1"}, ] [package.dependencies] @@ -2831,4 +2766,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "70dde474fd4580a2207cbeb0fdbe3c763891e0cd19317908e9b346fa5cb9f6b1" +content-hash = "8f623a140b1f5a63f967123172463d07d41c4269d840262fd6dbcd2f4bab4b6d" diff --git a/pyproject.toml b/pyproject.toml index 70f9847..89801b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,10 +34,11 @@ typing-extensions = "^4.12.0" loguru = "^0.7.2" httpx = "^0.27.2" markdownify = "^0.13.1" -litellm = "^1.52.9" +litellm = "^1.57.0" pillow = "^11.0.0" json-repair = "^0.30.1" tldextract = "^5.1.3" +anthropic = "^0.42.0" [tool.poetry.group.dev.dependencies] diff --git a/test.py b/test.py new file mode 100644 index 0000000..7df191e --- /dev/null +++ b/test.py @@ -0,0 +1,26 @@ +from dendrite import AsyncDendrite + + +async def send_email(to, subject, message): + client = AsyncDendrite(auth="outlook.live.com") + + # Navigate + await client.goto( + "https://outlook.live.com/mail/0/", expected_page="An email inbox" + ) + + # Create new email and populate fields + await client.click("The new email button") + await client.fill("The recipient field", to) + await client.press("Enter") + await client.fill("The subject field", subject) + await client.fill("The message field", message) + + # Send email + await client.press("Enter", hold_cmd=True) + + +if __name__ == "__main__": + import asyncio + + asyncio.run(send_email("charles@dendrite.systems", "Hello", "This is a test email")) From a341e828a3dcead9cdb1ec93d466b03ff871bf1b Mon Sep 17 00:00:00 2001 From: charlesmaddock Date: Mon, 6 Jan 2025 16:49:11 +0100 Subject: [PATCH 17/18] removed cache and test file --- .dendrite/cache/extract.json | 30 - .dendrite/cache/get_element.json | 231 ---- .dendrite/cache/storage_state.json | 1151 ------------------- .dendrite/test_cache/extract.json | 1 - .dendrite/test_cache/get_element.json | 11 - .dendrite/test_cache/storage_state.json | 1 - dendrite/.dendrite/cache/extract.json | 1 - dendrite/.dendrite/cache/get_element.json | 32 - dendrite/.dendrite/cache/storage_state.json | 1 - 9 files changed, 1459 deletions(-) delete mode 100644 .dendrite/cache/extract.json delete mode 100644 .dendrite/cache/get_element.json delete mode 100644 .dendrite/cache/storage_state.json delete mode 100644 .dendrite/test_cache/extract.json delete mode 100644 .dendrite/test_cache/get_element.json delete mode 100644 .dendrite/test_cache/storage_state.json delete mode 100644 dendrite/.dendrite/cache/extract.json delete mode 100644 dendrite/.dendrite/cache/get_element.json delete mode 100644 dendrite/.dendrite/cache/storage_state.json diff --git a/.dendrite/cache/extract.json b/.dendrite/cache/extract.json deleted file mode 100644 index efd0e39..0000000 --- a/.dendrite/cache/extract.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "506bbd9210aa4b0001c1bb6b746edb0c": [ - { - "url": "https://www.google.com/search?q=hello+world&sca_esv=56c33dbfb6047401&source=hp&ei=4-h7Z6OvN46pwPAPqt6U0Ac&iflsig=AL9hbdgAAAAAZ3v28yMXGMY0IGzLtj6wbRHxrZRtxR7m&ved=0ahUKEwjjrP6yp-GKAxWOFBAIHSovBXoQ4dUDCA4&uact=5&oq=hello+world&gs_lp=Egdnd3Mtd2l6IgtoZWxsbyB3b3JsZEgMUABYAHAAeACQAQCYAQCgAQCqAQC4AQPIAQD4AQGYAgCgAgCYAwCSBwCgBwA&sclient=gws-wiz", - "domain": "www.google.com", - "script": "# Find all search result containers and extract titles and urls\nresults = []\nfor result in soup.find_all('div', class_='yuRUbf'):\n link = result.find('a')\n if link:\n url = link.get('href')\n title = link.find('h3', class_='LC20lb').get_text() if link.find('h3', class_='LC20lb') else None\n if url and title:\n results.append({\n 'url': url,\n 'title': title\n })\nresponse_data = results", - "created_at": "2025-01-06T15:30:40.319245" - }, - { - "url": "https://www.google.com/search?q=hello+world&sca_esv=56c33dbfb6047401&source=hp&ei=OOl7Z5vyBPLGwPAPtLa8-Qs&iflsig=AL9hbdgAAAAAZ3v3SMS1-bHO8XE5tguRZxhUXc5KxEX9&ved=0ahUKEwib7o_bp-GKAxVyIxAIHTQbL78Q4dUDCA4&uact=5&oq=hello+world&gs_lp=Egdnd3Mtd2l6IgtoZWxsbyB3b3JsZEgKUABYAHAAeACQAQCYAQCgAQCqAQC4AQPIAQD4AQGYAgCgAgCYAwCSBwCgBwA&sclient=gws-wiz", - "domain": "www.google.com", - "script": "results = []\nfor result in soup.find_all('div', class_='yuRUbf'):\n link = result.find('a')\n if link:\n url = link.get('href')\n title = link.find('h3', class_='LC20lb')\n if title:\n title = title.get_text(strip=True)\n results.append({\n 'url': url,\n 'title': title\n })\nresponse_data = results", - "created_at": "2025-01-06T15:31:54.390225" - } - ], - "efcae8c2d66eb756a9d08869713c650d": [ - { - "url": "https://dendrite.systems/", - "domain": "dendrite.systems", - "script": "response_data = soup.find('div', {'class': 'chakra-stack css-11upoe7'}).prettify()", - "created_at": "2025-01-06T16:08:19.639139" - }, - { - "url": "https://dendrite.systems/", - "domain": "dendrite.systems", - "script": "response_data = soup.find('div', {'class': 'chakra-stack css-11upoe7'}).prettify()", - "created_at": "2025-01-06T16:19:11.196902" - } - ] -} \ No newline at end of file diff --git a/.dendrite/cache/get_element.json b/.dendrite/cache/get_element.json deleted file mode 100644 index ab287aa..0000000 --- a/.dendrite/cache/get_element.json +++ /dev/null @@ -1,231 +0,0 @@ -{ - "7b94db3c84c0214bb29f20c70efc4f86": [ - { - "selector": "div[id=\"news-item-0\"] > div:nth-child(3) > div:nth-child(3) > div > noscript > div", - "prompt": "link for all latest private buy and sell postings\n\nThe element should be clickable.", - "url": "https://www.sweclockers.com/", - "netloc": "www.sweclockers.com", - "created_at": "2025-01-06T15:09:51.410885" - } - ], - "47787fc6aa09b7b6445315d032a16e3a": [ - { - "selector": "body > noscript > meta", - "prompt": "Reject all cookies\n\nThe element should be clickable.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:10:26.905612" - }, - { - "selector": "body > noscript > meta", - "prompt": "Reject all cookies\n\nThe element should be clickable.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:11:19.229277" - }, - { - "selector": "body > noscript > meta", - "prompt": "Reject all cookies\n\nThe element should be clickable.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:11:24.921313" - }, - { - "selector": "button[id=\"W0wltc\"]", - "prompt": "Reject all cookies\n\nThe element should be clickable.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:30:08.823683" - } - ], - "c1094c675ffbbcc5a0d1f2c3f353f64e": [ - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:11:30.085245" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:11:35.239066" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:11:39.749348" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:11:44.421790" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:13:22.794447" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:13:27.643231" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:16:30.192815" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:16:34.778220" - }, - { - "selector": "textarea[id=\"APjFqb\"]", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:29:04.780560" - }, - { - "selector": "textarea[id=\"APjFqb\"]", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:30:14.829934" - }, - { - "selector": "textarea[id=\"APjFqb\"]", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:31:25.919497" - } - ], - "f68f83fb21a2d0573cc855ee8508cace": [ - { - "selector": "button[aria-label=\"New\\ mail\"]", - "prompt": "The new email button\n\nThe element should be clickable.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:26:49.927767" - } - ], - "6f7b4a48ea5909af9eca6ab2d0afaa60": [ - { - "selector": "div[aria-label=\"To\"]", - "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Recipient', 'Subject', 'Message']) in this page. Get the field best described as 'Recipient'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:27:01.932492" - } - ], - "2756844c43a183655e870811fe56f9e8": [ - { - "selector": "span.DeHTj", - "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Recipient', 'Subject', 'Message']) in this page. Get the field best described as 'Subject'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:27:14.555811" - } - ], - "b2c1bd5c5ec4557d93d091e581da8d20": [ - { - "selector": "div[aria-label=\"Message\\ body\\,\\ press\\ Alt\\+F10\\ to\\ exit\"]", - "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Recipient', 'Subject', 'Message']) in this page. Get the field best described as 'Message'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:27:26.121823" - } - ], - "e33e95dde2eacd1d3919fda132bd8f5a": [ - { - "selector": "button[id=\"splitButton-r24__primaryActionButton\"]", - "prompt": "The send button\n\nThe element should be clickable.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:27:36.927998" - }, - { - "selector": "button[id=\"splitButton-r1p__primaryActionButton\"]", - "prompt": "The send button\n\nThe element should be clickable.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:30:12.619521" - }, - { - "selector": "button[id=\"splitButton-r1e__primaryActionButton\"]", - "prompt": "The send button\n\nThe element should be clickable.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:43:14.041097" - } - ], - "f186d514ed4f8f795ca083b1cc661b05": [ - { - "selector": "div[aria-label=\"To\"]", - "prompt": "The recipient field\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:28:41.359533" - } - ], - "783d6e8491de54537999dd4a39069f99": [ - { - "selector": "span.DeHTj", - "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Subject', 'Message']) in this page. Get the field best described as 'Subject'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:29:42.113739" - } - ], - "9519aa7364cd93eedbb70dca73a7336e": [ - { - "selector": "div[aria-label=\"Message\\ body\\,\\ press\\ Alt\\+F10\\ to\\ exit\"]", - "prompt": "I'll be filling in text in several fields with these keys: dict_keys(['Subject', 'Message']) in this page. Get the field best described as 'Message'. I want to fill it with a '' type value.\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:29:53.726664" - } - ], - "17e46caa6a5819956a7c46d62b65ac0e": [ - { - "selector": "input[aria-label=\"Add\\ a\\ subject\"]", - "prompt": "The subject field\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:40:35.649464" - } - ], - "71d43663082637d3bbbd34d776d3bea4": [ - { - "selector": "div.elementToProof", - "prompt": "The message field\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:40:45.019897" - }, - { - "selector": "div[aria-label=\"Message\\ body\\,\\ press\\ Alt\\+F10\\ to\\ exit\"]", - "prompt": "The message field\n\nMake sure the element can be filled with text.", - "url": "https://outlook.live.com/mail/0/", - "netloc": "outlook.live.com", - "created_at": "2025-01-06T16:42:56.855528" - } - ] -} \ No newline at end of file diff --git a/.dendrite/cache/storage_state.json b/.dendrite/cache/storage_state.json deleted file mode 100644 index 626a3a0..0000000 --- a/.dendrite/cache/storage_state.json +++ /dev/null @@ -1,1151 +0,0 @@ -{ - "2813eeabfd22ab1b344ad7263486fd4d": [ - { - "origins": [ - { - "origin": "https://outlook.live.com", - "localStorage": [ - { - "name": "olk-dla", - "value": "1736176839937" - }, - { - "name": "olk-WebPushUserSettings", - "value": "[{\"userPreferenceIdentifier\":\"00037fff-8441-f3e2-0000-000000000000_\",\"lastSessionDate\":\"Mon Jan 06 2025\",\"uniqueDaysSessionCount\":1,\"enabled\":null,\"mailEnabled\":null,\"reminderEnabled\":null,\"vipMailEnabled\":null,\"lastCheckboxUnchecked\":null,\"promptCount\":0,\"lastPromptDate\":null}]" - }, - { - "name": "olk-UsersNormalizedThemeImage", - "value": "assets/mail/themes/modern/v2/arcticsolitude/light.jpg" - }, - { - "name": "olk-isTimeZoneCacheAvailable", - "value": "true" - }, - { - "name": "olk-ActionableMessages.ClientConfiguration_960e1055cb51450f0a444f174f48fd4ad9bec92b5f78966f6481c00645c7a8aa", - "value": "{\"data\":{\"organizationConfigs\":{\"ConnectorsEnabled\":true,\"ConnectorsActionableMessagesEnabled\":true,\"OutlookPayEnabled\":false,\"SmtpActionableMessagesEnabled\":true},\"providerConfigs\":[{\"originator\":\"5ED5E1C4-1023-46CD-8D17-F8942D0CD5DD\"},{\"originator\":\"62242E1D-C39A-4F8A-A66E-1D39C0A336B0\"},{\"originator\":\"569a3659-9ebb-4ffc-98c5-7796c9f9d695\"},{\"originator\":\"87ffcf3a-1b34-41ea-88ff-21eaae9870ee\"},{\"originator\":\"6d4f58eb-dcdd-4fe5-b554-82d9305ce7ee\"},{\"originator\":\"6acc43d6-4718-4d95-8f05-42311a7188a7\"}],\"providerConfigsDarkMode\":[{\"originator\":\"5ED5E1C4-1023-46CD-8D17-F8942D0CD5DD\"},{\"originator\":\"62242E1D-C39A-4F8A-A66E-1D39C0A336B0\"},{\"originator\":\"569a3659-9ebb-4ffc-98c5-7796c9f9d695\"},{\"originator\":\"87ffcf3a-1b34-41ea-88ff-21eaae9870ee\"},{\"originator\":\"6d4f58eb-dcdd-4fe5-b554-82d9305ce7ee\"},{\"originator\":\"6acc43d6-4718-4d95-8f05-42311a7188a7\"}],\"themeMappings\":{\"connectors\":{\"teams\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/teams/teams.json\",\"infobar-generic\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/infobar/infobar-generic.json\",\"infobar-alert\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/infobar/infobar-alert.json\",\"txp-demo\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/txp/txp-demo.json\",\"txp-opay\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/txp/txp-opay.json\",\"agendamail\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/agendamail/agendamail.json\",\"compact\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/compact/compact.json\",\"myanalytics\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/myanalytics/myanalytics.json\",\"ram-word\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-word.json\",\"ram-excel\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-excel.json\",\"ram-powerpoint\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-powerpoint.json\",\"reply_at_mentions-word\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-word.json\",\"reply_at_mentions-excel\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-excel.json\",\"reply_at_mentions-powerpoint\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-powerpoint.json\",\"cseo\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/cseo/cseo.json\",\"unifiedgroups\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/unifiedgroups/unifiedgroups.json\",\"cortana\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/cortana/cortana.json\",\"msteams\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/msteams/msteams.json\",\"surveymonkey\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/surveymonkey/surveymonkey.json\",\"officeforms\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/officeforms/officeforms.json\",\"datahygieneengine\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/datahygieneengine/datahygieneengine.json\"}},\"themeMappingsDarkMode\":{\"connectors\":{}},\"hostCapabilities\":{\"AADAuthentication\":\"*\",\"Transaction\":\"*\"},\"clientTimeouts\":{\"default\":{\"AutoInvokedActionTimeoutInMsec\":6000,\"HttpActionTimeoutInMsec\":15000,\"SnackBarTimeoutInMsec\":8000}},\"clientTimeoutsACv2\":{\"default\":{\"AutoInvokedActionTimeoutInMsec\":10000,\"HttpActionTimeoutInMsec\":20000,\"SnackBarTimeoutInMsec\":8000}}},\"state\":1,\"time\":1736176852482}" - }, - { - "name": "olk-bootstrapMailListItemWindowWidth", - "value": "1280" - }, - { - "name": "olk-sla", - "value": "0" - }, - { - "name": "olk-tour-state", - "value": "{\"lastClientActiveDate\":\"Mon, 06 Jan 2025 15:20:52 GMT\",\"tourEnabled\":false,\"tourCollapsed\":false,\"moduleTourStates\":{\"Mail\":{\"tourEnabled\":false,\"tourCollapsed\":false}}}" - }, - { - "name": "olk-sdfp", - "value": "{\"TimeZoneStr\":\"Greenwich Standard Time\",\"FolderPaneBitFlags\":0}" - }, - { - "name": "sessionTracking_00037FFF8441F3E2", - "value": "{\"authenticatedState\":1,\"upn\":\"dendrite_labs@outlook.com\",\"idp\":\"msa\",\"lastActiveTime\":1736176839463}" - }, - { - "name": "olk-MailOwaPreloadStrings", - "value": "[\"https://res.cdn.office.net/owamail/hashed-v1/scripts/../resources/locale/en/owa.AppBoot.m.ce09dc99.json\"]" - }, - { - "name": "olk-UsersNormalizedTheme", - "value": "arcticsolitude" - }, - { - "name": "olk-ActionableMessages.ActionEndpoint_960e1055cb51450f0a444f174f48fd4ad9bec92b5f78966f6481c00645c7a8aa", - "value": "{\"data\":{\"Url\":\"/actionsb2netcore\"},\"state\":2,\"time\":1736176852045,\"successiveCount\":1}" - }, - { - "name": "olk-BootDiagnostics", - "value": "{\"puid\":\"00037FFF8441F3E2\",\"tid\":\"84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa\",\"mbx\":\"00037fff-8441-f3e2-0000-000000000000\",\"prem\":\"0\",\"isCon\":true,\"upn\":\"dendrite_labs@outlook.com\"}" - }, - { - "name": "olk-LogicalRing", - "value": "WW" - }, - { - "name": "olk-mail_conditionalformattingdendrite_labs@outlook.com", - "value": "[]" - }, - { - "name": "olk-UnifiedConsentRequestData", - "value": "{\"id\":\"c147f8be-56d9-452c-b20e-b105df5eba21_ucsisunotice_1\",\"modelType\":\"ucsisunotice\",\"status\":\"read\",\"version\":\"1.14\",\"consentType\":\"prominentnotice\",\"maxPromptsReached\":true,\"consentedBy\":\"user\",\"consentedInSurface\":\"appstart\",\"consentedUTC\":\"2024-10-17T09:59:13.1833406+00:00\",\"values\":null,\"needConsent\":false,\"consentReason\":\"unknown\",\"lastRequestedConsentTimeInMsUTC\":1736176846100}" - }, - { - "name": "olk-bootstrapMailListItemViewSwapSetting", - "value": "true" - }, - { - "name": "olk-EnvDiagnostics", - "value": "{\"fe\":\"BN0PR04CA0083, GV3P280CA0102\",\"be\":\"BN8PR03MB4850\",\"wsver\":\"15.20.8314.18\",\"fost\":\"NAMPRD03\",\"dag\":\"NAMPR03DG090\",\"te\":\"0\"}" - }, - { - "name": "olk-OwaClientId", - "value": "F92C019C4DC04A9A8B76EE06C7E7B683" - }, - { - "name": "olk-slaef", - "value": "-1" - }, - { - "name": "O365Shell_ECS_config", - "value": "{\"9897789ed7282bc5e2a60091cd781bb2b482e27793b0479995f1209675fa3ec3\":{\"expiryTime\":1736263238873,\"value\":{\"OneShell\":{\"UpdatedConsumerAppList\":true,\"M365StartEnabled\":true,\"DisableM365StartIntentsModule\":false,\"default\":true},\"Headers\":{\"ETag\":\"\\\"8M6C3IBLtb8mwT1KNreplkZ/i0rFbbeiyWPkxzwWrg0=\\\"\",\"Expires\":\"Mon, 06 Jan 2025 16:20:38 GMT\",\"CountryCode\":\"SE\",\"StatusCode\":\"200\"},\"ConfigIDs\":{\"OneShell\":\"P-R-1157040-4-8,P-R-1131228-4-17,P-D-1117449-1-4\"}}}}" - }, - { - "name": "sessionTracking_SignedInAccountList", - "value": "[{\"sessionTrackingKey\":\"sessionTracking_00037FFF8441F3E2\",\"lastActiveTime\":1736176839463}]" - }, - { - "name": "olk-dlaef", - "value": "-1" - }, - { - "name": "olk-leftNavAtV2DisplayDate", - "value": "2025-01-06T15:20:40.091Z" - }, - { - "name": "olk-OwaLocale", - "value": "en" - }, - { - "name": "olk-OwaSessionCount", - "value": "1" - }, - { - "name": "olk-undefinedOwaPreloadStrings", - "value": "[\"https://res.cdn.office.net/owamail/hashed-v1/scripts/../resources/locale/en/owa.worker.data.b526d83d.json\"]" - }, - { - "name": "olk-ActionableMessages.ClientConfiguration", - "value": "{\"data\":{\"organizationConfigs\":{\"ConnectorsEnabled\":true,\"ConnectorsActionableMessagesEnabled\":true,\"OutlookPayEnabled\":false,\"SmtpActionableMessagesEnabled\":true},\"providerConfigs\":[{\"originator\":\"5ED5E1C4-1023-46CD-8D17-F8942D0CD5DD\"},{\"originator\":\"62242E1D-C39A-4F8A-A66E-1D39C0A336B0\"},{\"originator\":\"569a3659-9ebb-4ffc-98c5-7796c9f9d695\"},{\"originator\":\"87ffcf3a-1b34-41ea-88ff-21eaae9870ee\"},{\"originator\":\"6d4f58eb-dcdd-4fe5-b554-82d9305ce7ee\"},{\"originator\":\"6acc43d6-4718-4d95-8f05-42311a7188a7\"}],\"providerConfigsDarkMode\":[{\"originator\":\"5ED5E1C4-1023-46CD-8D17-F8942D0CD5DD\"},{\"originator\":\"62242E1D-C39A-4F8A-A66E-1D39C0A336B0\"},{\"originator\":\"569a3659-9ebb-4ffc-98c5-7796c9f9d695\"},{\"originator\":\"87ffcf3a-1b34-41ea-88ff-21eaae9870ee\"},{\"originator\":\"6d4f58eb-dcdd-4fe5-b554-82d9305ce7ee\"},{\"originator\":\"6acc43d6-4718-4d95-8f05-42311a7188a7\"}],\"themeMappings\":{\"connectors\":{\"teams\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/teams/teams.json\",\"infobar-generic\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/infobar/infobar-generic.json\",\"infobar-alert\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/infobar/infobar-alert.json\",\"txp-demo\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/txp/txp-demo.json\",\"txp-opay\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/txp/txp-opay.json\",\"agendamail\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/agendamail/agendamail.json\",\"compact\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/compact/compact.json\",\"myanalytics\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/myanalytics/myanalytics.json\",\"ram-word\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-word.json\",\"ram-excel\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-excel.json\",\"ram-powerpoint\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/ram-powerpoint.json\",\"reply_at_mentions-word\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-word.json\",\"reply_at_mentions-excel\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-excel.json\",\"reply_at_mentions-powerpoint\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/ram/reply_at_mentions-powerpoint.json\",\"cseo\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/cseo/cseo.json\",\"unifiedgroups\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/unifiedgroups/unifiedgroups.json\",\"cortana\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/cortana/cortana.json\",\"msteams\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/msteams/msteams.json\",\"surveymonkey\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/surveymonkey/surveymonkey.json\",\"officeforms\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/officeforms/officeforms.json\",\"datahygieneengine\":\"https://res.cdn.office.net/actionablemessages/worcester/connectors/1.0.803.0/actions/themes/owa/datahygieneengine/datahygieneengine.json\"}},\"themeMappingsDarkMode\":{\"connectors\":{}},\"hostCapabilities\":{\"AADAuthentication\":\"*\",\"Transaction\":\"*\"},\"clientTimeouts\":{\"default\":{\"AutoInvokedActionTimeoutInMsec\":6000,\"HttpActionTimeoutInMsec\":15000,\"SnackBarTimeoutInMsec\":8000}},\"clientTimeoutsACv2\":{\"default\":{\"AutoInvokedActionTimeoutInMsec\":10000,\"HttpActionTimeoutInMsec\":20000,\"SnackBarTimeoutInMsec\":8000}}},\"state\":1,\"time\":1736176852482}" - }, - { - "name": "O365Shell_ThemeInfo_Consumer", - "value": "{\"Id\":\"arcticsolitude\",\"Primary\":\"#0F6CBD\",\"NavBar\":\"transparent\",\"DefaultBackground\":\"transparent\",\"BackgroundImageUrl\":null,\"DefaultText\":\"#242424cc\",\"AppName\":\"#242424\",\"FullBleedImages\":null,\"userPersonalizationAllowed\":true}" - }, - { - "name": "olk-sdmp", - "value": "{\"TimeZoneStr\":\"Greenwich Standard Time\",\"InboxReadingPanePosition\":1,\"IsFocusedInboxOn\":true,\"BootWithConversationView\":true,\"SortResults\":[{\"Path\":{\"__type\":\"PropertyUri:#Exchange\",\"FieldURI\":\"conversation:LastDeliveryOrRenewTime\"},\"Order\":\"Descending\"},{\"Path\":{\"__type\":\"PropertyUri:#Exchange\",\"FieldURI\":\"conversation:LastDeliveryTime\"},\"Order\":\"Descending\"}],\"IsSenderScreeningSettingEnabled\":false}" - }, - { - "name": "olk-ActionableMessages.ActionEndpoint", - "value": "{\"data\":{\"Url\":\"/actionsb2netcore\"},\"state\":2,\"time\":1736176852045,\"successiveCount\":1}" - }, - { - "name": "olk-SmtpActionableMessagesEnabled_dendrite_labs@outlook.com", - "value": "true" - }, - { - "name": "olk-bootFailureCount", - "value": "0" - } - ] - } - ], - "cookies": [ - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/orgexplorer/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/platformapp/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/orgexplorer/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/platformapp/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/orgexplorer/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/platformapp/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/orgexplorer/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/platformapp/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/bookwithme/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/bookwithme/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/bookwithme/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/bookwithme/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/calendar/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/calendar/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/calendar/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/calendar/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/people/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/photos/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/spaces/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/mailb2/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/people/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/photos/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/spaces/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/mailb2/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/people/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/photos/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/spaces/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/mailb2/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/people/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/photos/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/spaces/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/mailb2/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/files/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/files/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/files/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "ConnectorsLtiToken", - "value": "eyJhbGciOiJSUzI1NiIsImtpZCI6IkEzMDVCMkU1Q0ZERjFGQTFBODgyNTU2MzM3NDhCQkNBRTAxNUU5OTIiLCJ0eXAiOiJKV1QiLCJ4NXQiOiJvd1d5NWNfZkg2R29nbFZqTjBpN3l1QVY2WkkifQ.eyJvaWQiOiIwMDAzN2ZmZi04NDQxLWYzZTItMDAwMC0wMDAwMDAwMDAwMDAiLCJwdWlkIjoiMDAwMzdGRkY4NDQxRjNFMiIsInNtdHAiOiJkZW5kcml0ZV9sYWJzQG91dGxvb2suY29tIiwiY2lkIjoiNUQ5OEEwQTY1QjkxNDk4QyIsInZlciI6IkV4Y2hhbmdlLkNhbGxiYWNrLlYyIiwiYXBwaWQiOiI0MGI4NTJkZi05NjkzLTQyMGQtYWE3ZC0xMDA3MmM5Y2UwNzciLCJkZXBsb3ltZW50aWQiOiJodHRwczovL291dGxvb2sub2ZmaWNlMzY1LmNvbS8iLCJ0aWQiOiI4NGRmOWU3Zi1lOWY2LTQwYWYtYjQzNS1hYWFhYWFhYWFhYWEiLCJhY3IiOiIxIiwiYXBwaWRhY3IiOiIwIiwic2NwIjoiQ29ubmVjdG9ycy5NYW5hZ2VtZW50LldlYiIsIm5iZiI6MTczNjE3Njg1MiwiZXhwIjoxNzM2MTc3NzUyLCJpc3MiOiJodHRwczovL291dGxvb2sub2ZmaWNlMzY1LmNvbS8iLCJhdWQiOiJodHRwczovL291dGxvb2sub2ZmaWNlLmNvbSIsImhhcHAiOiJvd2EifQ.H1tXgOQdUGkqTaIbrGxwmBopFXiX21_UV-c2AaflhJxXa6AE2IGRgrZqbc5IhM296hsckXyEUqk7LzRtS0IZcT54DiokmIpmNQQ_8b_4KPy5WwrgtBDJPSwPJekfNusSN-L4CHiOS6oVq8gqQrQ16pjOZOUCrON5pcR1RpeJ6WClEovUAjv69ZGyxm9QMEw0cK6UvSZeDUvmUsMZpzs8spnO-dxGKHf_PmAz065LV3W3x3BwyuRBUzgOPYwE7T_Vr1US6qiQDQjyBYwxnF5IgKCqM-aQku7UJCVEY7-hDqw6YpjkCVvlZRb8QX8dZ9lFTl-N7arnVi5Z2QS46VUQ6w", - "domain": "outlook.live.com", - "path": "/actions/", - "expires": -1, - "httpOnly": false, - "secure": false, - "sameSite": "Lax" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/files/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/mail/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/host/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/meet/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/feed/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/mail/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/host/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/meet/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/feed/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/mail/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/host/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/meet/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/feed/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/mail/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/host/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/meet/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/feed/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "DefaultAnchorMailbox", - "value": "00037FFF8441F3E2@84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa", - "domain": "outlook.live.com", - "path": "/owa/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "O365Consumer", - "value": "1", - "domain": "outlook.live.com", - "path": "/owa/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SuiteServiceProxyKey", - "value": "PRPuViFNJ2G0fnUS5vCq5L7UXZ7gd+x7FBPOVmMgKyw=&Nl03zDlWY/m0GC2qtk/Oag==", - "domain": "outlook.live.com", - "path": "/owa/0/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "MicrosoftApplicationsTelemetryDeviceId", - "value": "cab7afd7-5310-423d-adeb-bf6a43c995c1", - "domain": "outlook.live.com", - "path": "/mail/0", - "expires": 1767712852, - "httpOnly": false, - "secure": false, - "sameSite": "Lax" - }, - { - "name": "MicrosoftApplicationsTelemetryFirstLaunchTime", - "value": "2025-01-06T15:20:52.066Z", - "domain": "outlook.live.com", - "path": "/mail/0", - "expires": 1767712852, - "httpOnly": false, - "secure": false, - "sameSite": "Lax" - }, - { - "name": "X-OWA-CANARY", - "value": "wTQ4Tc3OklgAAAAAAAAAAHAzJrVlLt0Yc9EFI5IZSTUAuscT6___E4cuXtNv9DBKVaZkGGn1Aw8.", - "domain": "outlook.live.com", - "path": "/owa/0/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "ClientId", - "value": "F92C019C4DC04A9A8B76EE06C7E7B683", - "domain": "outlook.live.com", - "path": "/", - "expires": 1767712795.734867, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSPBack", - "value": "0", - "domain": ".login.live.com", - "path": "/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "exchangecookie", - "value": "b32957dafaba492f98a9062fe2472cd8", - "domain": "outlook.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "RpsCsrfState.W7_rqj9rYnRPOvCY0n5Lhs6vO06f66uJX8aL2DyFR3U", - "value": "59cdafb6-3962-e10d-510d-e14b1a5bd5c1", - "domain": "outlook.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSCC", - "value": "88.131.152.178-SE", - "domain": ".login.live.com", - "path": "/", - "expires": 1769872798.800552, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "MicrosoftApplicationsTelemetryDeviceId", - "value": "9b74c3ff-d18e-404d-a664-f01ee22c8c9e", - "domain": "login.live.com", - "path": "/", - "expires": 1767712834.773738, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "fptctx2", - "value": "taBcrIH61PuCVH7eNCyH0FWPWMZs3CpAZMKmhMiLe%252bEHNIBLAHqWFybWaDSk%252bsssqnNNUqnhfYAnMHGshIu0%252bjXl9qScCvWinzgWgiL%252bFYOdabJKXklG1D6Hk8yNrTt6GjEx4%252fmDTbsQIseN%252fnFY5WX2XfvhARX640Pg0rSmBBVOsEeU1MUIAY9GEOI4idGM62Vor91JDF3k3y8TLHsw8Tks24iOWupX%252f6NYrU8awu9uRLDBbe%252fzR3wsb3XaXcnundPGCwes7VczQDwgLzTn1wHsbzcJySiR4HaQLJ0HMAFhQbsiOiy9Yw5T7%252bKojDmysP0n4PAwyhyolmr0eha4tw%253d%253d", - "domain": ".live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "Lax" - }, - { - "name": "MSFPC", - "value": "GUID=e82e1b9d7c7c40788ec16bf278e17420&HASH=e82e&LV=202501&V=4&LU=1736176799208", - "domain": "login.live.com", - "path": "/", - "expires": 1767712803.240902, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "PPLState", - "value": "1", - "domain": ".live.com", - "path": "/", - "expires": 1769872846.66053, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "ai_session", - "value": "zALKfrSoOmqE1KVXys7eui|1736176799114|1736176834880", - "domain": "login.live.com", - "path": "/", - "expires": 1736178634, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSPOK", - "value": "$uuid-ed5ca974-dac6-4f3d-a99c-b4c73611ba60$uuid-013dfd04-9ec5-4686-b50d-7ae7e98fd6b4", - "domain": ".login.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSPPre", - "value": "dendrite_labs%40outlook.com%7c5d98a0a65b91498c%7c%7c", - "domain": ".login.live.com", - "path": "/", - "expires": 1769872838.119016, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSPCID", - "value": "5d98a0a65b91498c", - "domain": ".login.live.com", - "path": "/", - "expires": 1769872846.660561, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSPAuth", - "value": "Disabled", - "domain": ".live.com", - "path": "/", - "expires": 1769872838.119111, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSPProf", - "value": "Disabled", - "domain": ".live.com", - "path": "/", - "expires": 1769872838.119162, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "NAP", - "value": "V=1.9&E=1e56&C=Rn7_DxqIlYAE2TP5QPR538qSgaQtg3RHjK_azveARr5BW9RwChqvEQ&W=1", - "domain": ".live.com", - "path": "/", - "expires": 1744842038.165454, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "ANON", - "value": "A=59A764CC7FC76B140B47E186FFFFFFFF&E=1eb0&W=1", - "domain": ".live.com", - "path": "/", - "expires": 1753482038.16544, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "WLSSC", - "value": "EgAuAgMAAAAMgAAAqAABbz75A45XrIQgHJ9K1NwFTgoY4641GLZglltOVU0vAOgYECkIWy2kdzc1Zip4MNzSKeCexmaq7TXRLNnzZzEpobFUU+9PKODnlRlq1ipfdvohueLyjvSSUPPpd/wQpl1liyBQs6iS8bpF9i4S2txA9C8UmRCN0Ucl/xdx4uPhJRUd9sU+LZJ4jGy8mlnWbkTEPVz4KXTZhKdWneAewpgQwrb148LG44sCPx4ia8wjq92Z08LQiYCQb78nYV1KL6CQbuoTAYWYEKHvRLECPdZZrerbB1tj0wnC7Ff1dnOTYmSf23GI4i/ZWmYHwKnBYpD+fcCvqAJogqHeISATgXV1lR0BfgAdAf9/AwDi80GExfR7Z8L0e2cQJwAAChCggBAaAGRlbmRyaXRlX2xhYnNAb3V0bG9vay5jb20AXAAAJmRlbmRyaXRlX2xhYnMlb3V0bG9vay5jb21AcGFzc3BvcnQuY29tAAAAA1VTAAAAAAAABAkCAACPp1VAAAZDAAZkZW5kcmkAAnRlAAAAAAAAAAAAAAAAAAAAAAAAW5FJjF2YoKYAAMX0e2fCm/JnAAAAAAAAAAAAAAAADwA4OC4xMzEuMTUyLjE3OAAFAQAAAAAAAAAAAAAAAAEEAAAAAAAAAAAAAAAAAAAAymcAWWR/tQsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAA", - "domain": ".live.com", - "path": "/", - "expires": 1769872838.119594, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "SDIDC", - "value": "CjRY4d4t5GHxoqC*X1II0hjuG8ZudjtJoQVoSzQ5lLHHK9UfiU1FRSajmJGP3*beMk8YZd5HG4J!aYdcShnQ2rcMWNfMvqDrlHfdC!2c5v5WpcqrzVVeFZA!v9QK!9aIiA$$", - "domain": ".login.live.com", - "path": "/", - "expires": 1769872838.11964, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "JSHP", - "value": "3$dendrite_labs%40outlook.com$dendri$te$$2$0$0$6413239986853085028$0", - "domain": ".login.live.com", - "path": "/", - "expires": 1769872846.660633, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "JSH", - "value": "3$dendrite_labs%40outlook.com$dendri$te$$2$0$0$6413239986853085028$0", - "domain": ".login.live.com", - "path": "/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSPSoftVis", - "value": "@72198325083833620@:@", - "domain": ".login.live.com", - "path": "/", - "expires": 1769872838.119817, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "orgName", - "value": "outlook.com", - "domain": "outlook.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "domainName", - "value": "", - "domain": "outlook.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "LI", - "value": "1:S171tHWqeVrL%2bzkcSS0RqY66hIEoMVwIAST92PQh5pM%3d:H4sIAAAAAAAEAGNkYGBgZIAAViib0QDEk0xJzUspyixJjc9JTCp2yC8tycnPz9ZLzs9lAsoKG5jpGxjqGxkYmSoYmloZGVgZmzOD9BqyAEmWkKLSVACrEe9GWwAAAA==", - "domain": "outlook.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "RPSSecAuth", - "value": "FAB2ARSz06Q8p88COKo8sjogMovqkAnFGw5mAAAMgAAAEBwG%2BWt4GLa6EgnIzOLrZ5QgAdoo5%2BdRJLUJfotKOwHycWU5VOZLlO0y25zEASfc81vdQheA8MLfDPybgor7OlskrfFWKSNB8mMjMeOCIqbjwk9hoDqZ6je93JKsKYAGz/Y7FGEjx3/qDLlw0ZtVN%2BnI5cbACEGr3RypygA/AWSzvHa/xM3Gkj31plaeLP4Nw7qbVoTApbx0x/2LxqO0iX9GRuTf9jhw%2Brq2I1uHjMGNE3j4OlcyHZ5VdcrW5FAl2c/baFlzIzEs20U5maotxpaW5dOGEBkwujb7EmygjmKyEm8ZhBM4W6pbf8NwyWNyvMLpvX4JRPqXifgeMTDoNiApBKxyluzvLj0NnYmlx0Nlz8Ar00R8OzE4pStN2NYuesolbGywp3V/JDZSKWu8woDS8iAA%2B3zgzQX7hZXhb/DBGFkN9TlS5Aiq30ehLzP%2BL6PkwT4%3D", - "domain": ".live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "MH", - "value": "MSFT", - "domain": ".live.com", - "path": "/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "UC", - "value": "e5548faecf0346e9850f44c2d29a398c", - "domain": "outlook.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "Lax" - }, - { - "name": "RoutingKeyCookie", - "value": "v2:H2Q6aEo3iWZU4gQEaoMcda8L4ajaSXKSwSfutOKc3oE%3d:00037fff-8441-f3e2-0000-000000000000@outlook.com", - "domain": "outlook.live.com", - "path": "/", - "expires": 1738768837.442808, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "X-OWA-RedirectHistory", - "value": "AkWK5rsBRfxRrGUu3Qg|AsmKpuYBZuYxrGUu3Qg|AhR7n8MBy48XlWUu3Qg|AhrYdSQBl2znkmUu3Qg", - "domain": "outlook.live.com", - "path": "/", - "expires": 1736198557.444199, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "ShCLSessionID", - "value": "1736176838297_0.2452314684191801", - "domain": "outlook.live.com", - "path": "/", - "expires": -1, - "httpOnly": false, - "secure": false, - "sameSite": "Lax" - }, - { - "name": "SM", - "value": "C", - "domain": ".c.live.com", - "path": "/", - "expires": -1, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "MUID", - "value": "39F11D600E5861AB3DB0080C0F7060A2", - "domain": ".live.com", - "path": "/", - "expires": 1769872839.698497, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "SRM_L", - "value": "39F11D600E5861AB3DB0080C0F7060A2", - "domain": ".c.live.com", - "path": "/", - "expires": 1769872839.698539, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "MR", - "value": "0", - "domain": ".c.live.com", - "path": "/", - "expires": 1736781639.698557, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "ANONCHK", - "value": "1", - "domain": ".c.live.com", - "path": "/", - "expires": 1736177439.698573, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSFPC", - "value": "GUID=e82e1b9d7c7c40788ec16bf278e17420&HASH=e82e&LV=202501&V=4&LU=1736176799208", - "domain": "outlook.live.com", - "path": "/", - "expires": 1767712841.237023, - "httpOnly": false, - "secure": true, - "sameSite": "None" - }, - { - "name": "MSPRequ", - "value": "id=N<=1736176845&co=0", - "domain": ".login.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "uaid", - "value": "2a6c3510907e4d03865788fb8b9e86a6", - "domain": ".login.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "__Host-MSAAUTHP", - "value": "11-M.C555_SN1.0.U.CgT9yBmtfX!MolWa0RNVFEaD3YKIeX97uBbCcpo5LT4zq*!uaVErnmGCQzzi!xgcTB4MGhTP9jOhPn91hkT*sCObALKUkS!4wNJBoAvRrmEfAomk!jFAX4h2QgTxUer88!7*dPl!lUt8CCqZX09MDKKUQTBbGqaCAfLCVHgotLim95NJBYEl9rOxVQTjNMKuXr6kUKqND7Y4zYeXJz26Re5JbqYkSQnUgMij2Uws6NH9EQ3rZer08*TwAuFZB6E8tGxKDdoqe945ER0Uo5ed!Nf7SY77qcjO!GzwHHxo35TB", - "domain": "login.live.com", - "path": "/", - "expires": 1769872846.660508, - "httpOnly": true, - "secure": true, - "sameSite": "None" - }, - { - "name": "OParams", - "value": "11O.DggSuOYHeYc8yw53azHyaYZtpkLUm1ENd0gc4tee0dOyzVqzFGCsuBUquUDeQZyiKl!3LcgEg9MmaakouTCa5SvYmB80LT7S1Wb6eIavH0SDk61YFlUky0P6BUMOzfO3GUjuteSGh*jl6i4BZ1C6EjaGQv7o2DBFd6FIfUiP5L7JK72Gfoffx6eHytDXwfhiMB3HMadsDkCx6ENaMHb4z!0F2XvFMPTWIHsIQxGByeMg79tEWVPOnZiUV728NYJUGs6roGZLpBApKii9KgaKrOXB4lnLoGsrGbyfVjZSoatrJ7F*PLVui!KS3R2l53vCYQ9lZoo2vdgSUPVFY3rxi578nPvpB87KohWX5vQZ6MQ5UKm8EfM3shhopCS2qUWtJz*4Qn44c5oFRYxIHjO6a*SQ3DmUNbXcpeZtOTinX8misorsDrCSCFb4AQpCXUIDKdyq9PAqnNv*qmNQDIMCNujcygHdvYBZVeremBuvZsl4uH*qd1a!h!c68en8zwb6wEY2jMN8SZ1ZtcwvRdcVD1W12uwlPeCmUEoP*qXLisHMdSDVBmoHA*M7MPO4Wpy9ghpJttnd*8DubMwgp6lPvmGWPPIgn*SVKYRHUQlgOYBJeowzFI!6LTn7xLo7keRsToROl!whFoyR9!ulLI60SiG!fMoOJdKdlbtsehv9KDz5xGIS8HIqxgbLfNl0mgEiifAgsyP7bBokxc1n8yQcHYv5iT1g7sb!KpNdCeYmYw!uwtWyoIYFQI5CjI9d96pBmwRMvMYnEEoR!3c!GnK4yL8OmNzpUNbEL!JM*XHf0idcnxY3KeXaKtOCylAJl7QptU9pHw18NghE5hRAnocry3pE!UcjaPOhuUMLCnuZO4BcYfTBfNpE7xRlFP0Yqga6N1Q5bCkaVAzza3UL7Fox6XW1Ug*F!gHzQa2dmJDwSAutFNGR2uoWWrcKV5VE4xx5fTFvHleaDBGPe1KYXv94o0dDcFYLcxX6x7jq!vJyvb0cu7ir3hZ*9bwLvRhnr175TXmcSJpCagbsWFYQbLnmkE3uwfGazBeRc6FMtKtYrEgEEQ*35ZRczJb*uyL8pbIhplqm*3Dof4iyB!mXFpGG8B0F3a!rYQM37CLHeHGCRsE*ZCPsyoNvShJxStyB9Pr5CD3jp7tpx6HD0eBDuq!UMhj15mRpjMLhy1XnMeaGwITvyR3!ZmEur4*xslqykbvXHI4nTIis!4224KY1zYq88mqYDSJMeqoeJpijzhLHZcZrCI6VGivr4O0gJ35y!Tdrg6qfdUtd3ekUGA9oUcmHcWDFf2hWqFRU9Fxlw8nRDZ7rp!fWb9eJsS1sh7ml8YssvavXK6!6Po6uLJLVC3ockPWAyOnIj69yrA!B1w4D5LJOildumPYv7XFaVN9g23sJNj*Xeqz*WJAPSOR7wt3SJfKaNeds0SvszwCWM7dsr95SXAv4Kyxx8m01CtQUmMcowdHQTkJb9t8fnK0JlAC*Fm!vhcBItBaJ!pU9T!Gpa6gyxa5gimkmRu2q2ExlvT0zYWSwH*jlWlUlsE3P8KV*7EmqH3nlHRwEGOu3jWXeOmxVVftCKG!aSrYQiTUm2nnV7BJrI!NH2RtefbQP6vYleQA8GsyZx9KfQRdcY*Cv9ygWxs7*LlC2b3qFObE!H2X9owgmGfe7bSxZ678n2OQoo39BBEJmv5hoLGdlhv7Q5mUoYotzt6EHIZM3XFcIRkT83PmMzhedhnY6mORKxeEuwyDdvWs2mQLWa7qtRC0eDOhELgfaRDfX2DAm9QoUoUUjFXdzVl3aTWpOFJUPhXcjKTioN7Rm7TFxmpbLNe*TrWBT7zBZyI9X*jXKYvmS!KkpQ4gTG9Tp3wt8RP8R3n8uFiu6S3DnNRi64Sb3nsKbOaUig3a4!OzdzI1plfH0CLBt7uk6K1YPLaBuFCvfB38psooGNq!TAfsPnvBu7VAEsC6BCTfDyPrZNZpwXwLarTKMle3HrwHdrxJtXsWz!GZ*mxYaKxm4xWHkxP6fivGgFOUAsgx8PApfpIQI2rJDghLYNHwMqYWgpnrHp0XVAOCE8qxEtJX0elaxwJmcR595DGrcqc6Jm3n!H3LR!*Yj*KrZqkQLgbxvk4GdKqPdVNAowY7lexGi2hmDcdkdUh96Qk3nWJo034wEMMl!RxuDHPYVSH3lrEd3E6TgreaopYmyDHIl3hAgHqjrCNMiuuhBkxJQ96V5Uk5KSN*VwBhgb0nAaSwCsOmPxTvwBKZae*UxN62ezH98gwJOyKWsjcxXUuNl6J1kDwjnOd5PaGDsXkUG2zaeDN3KIdzluYvNBylK8RDUDlwLLlRjDLyrYZO2OMZlzOJkzAxFjL8JP*r2yNWUZ9xyzGG88EYxWuDQhy6dcNwKIawm8hrz1QqLVmj!*KhonfTzvgme1kWo12vcNCdEjcphBSMdREAfw6rpD8VVKkuCuTsLR6176HcGiD3WrMLJpdZZlHS6BVwb3d2AqtU4Z0fhjzmC3QXAPzpV4JT4kxnj23iu1H47jznCi2R7FTv!QcDzq1LEG9nEVQ*M!MDmZIWUgnLb9gglIvhe3MlgUBA$", - "domain": ".login.live.com", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - } - ] - } - ] -} \ No newline at end of file diff --git a/.dendrite/test_cache/extract.json b/.dendrite/test_cache/extract.json deleted file mode 100644 index 9e26dfe..0000000 --- a/.dendrite/test_cache/extract.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/.dendrite/test_cache/get_element.json b/.dendrite/test_cache/get_element.json deleted file mode 100644 index 8c17f56..0000000 --- a/.dendrite/test_cache/get_element.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "7b94db3c84c0214bb29f20c70efc4f86": [ - { - "selector": "div[id=\"news-item-0\"] > div:nth-child(3) > div:nth-child(3) > div > noscript > div", - "prompt": "link for all latest private buy and sell postings\n\nThe element should be clickable.", - "url": "https://www.sweclockers.com/", - "netloc": "www.sweclockers.com", - "created_at": "2025-01-06T15:09:16.487844" - } - ] -} \ No newline at end of file diff --git a/.dendrite/test_cache/storage_state.json b/.dendrite/test_cache/storage_state.json deleted file mode 100644 index 9e26dfe..0000000 --- a/.dendrite/test_cache/storage_state.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/dendrite/.dendrite/cache/extract.json b/dendrite/.dendrite/cache/extract.json deleted file mode 100644 index 9e26dfe..0000000 --- a/dendrite/.dendrite/cache/extract.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/dendrite/.dendrite/cache/get_element.json b/dendrite/.dendrite/cache/get_element.json deleted file mode 100644 index 91a7478..0000000 --- a/dendrite/.dendrite/cache/get_element.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "c1094c675ffbbcc5a0d1f2c3f353f64e": [ - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:20:12.336490" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:20:16.831358" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:20:20.718020" - }, - { - "selector": "body > noscript > meta", - "prompt": "Search input field\n\nMake sure the element can be filled with text.", - "url": "https://www.google.com/", - "netloc": "www.google.com", - "created_at": "2025-01-06T15:20:24.813117" - } - ] -} \ No newline at end of file diff --git a/dendrite/.dendrite/cache/storage_state.json b/dendrite/.dendrite/cache/storage_state.json deleted file mode 100644 index 9e26dfe..0000000 --- a/dendrite/.dendrite/cache/storage_state.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file From 177411d5bab4d29a109143b826aa24396ec413d2 Mon Sep 17 00:00:00 2001 From: Charles Maddock Date: Mon, 6 Jan 2025 09:13:55 -0800 Subject: [PATCH 18/18] Update README.md --- README.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2d885d4..0fdf3b9 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -> **Notice:** The Dendrite SDK is not under active development from us. However, the project will remain fully open source so that you and others can learn from and build upon our work. Feel free to fork, study, or adapt this code for your own projects as you wish – reach out to us on Discord if you have questions! -

    Dendrite Homepage Docs Discord

    +> **Notice:** The Dendrite SDK is not under active development anymore. However, the project will remain fully open source so that you and others can learn from it. Feel free to fork, study, or adapt this code for your own projects as you wish – reach out to us on Discord if you have questions! We love chatting about web AI agents. 🤖 + ## What is Dendrite? #### Dendrite is a framework that makes it easy for web AI agents to browse the internet just like humans do. Use Dendrite to: @@ -19,7 +19,9 @@ #### A simple outlook integration -With Dendrite, it's easy to create web interaction tools for your agent. +With Dendrite it's easy to create web interaction tools for your agent. + +Here's how you can send an email: ```python from dendrite import AsyncDendrite @@ -51,13 +53,24 @@ if __name__ == "__main__": ``` -To authenticate on outlook, run the command below: +You'll need to add your own Anthropic key or [configure which LLMs to use yourself](https://docs.dendrite.systems/concepts/config). -```bash -dendrite auth --url outlook.live.com + +```.env +ANTHROPIC_API_KEY=sk-... ``` -A browser will open and you'll be able to login. After you've logged in, press enter in your terminal to save the cookies locally, so that they can be used in your code. +To **authenticate** on any web service with Dendrite, follow these steps: + +1. Run the authentication command + + ```bash + dendrite auth --url outlook.live.com + ``` + +2. This command will open a browser that you'll be able to login with. + +3. After you've logged in, press enter in your terminal. This will save your cookies locally so that they can be used in your code. Read more about authentication [in our docs](https://docs.dendrite.systems/examples/authentication). @@ -69,15 +82,11 @@ pip install dendrite && dendrite install #### Simple navigation and interaction -Initialize the Dendrite client and start doing web interactions without boilerplate. - -[Get your API key here](https://dendrite.systems/app) - ```python from dendrite import AsyncDendrite async def main(): - client = AsyncDendrite(dendrite_api_key="sk...") + client = AsyncDendrite() await client.goto("https://google.com") await client.fill("Search field", "Hello world") @@ -94,7 +103,7 @@ In the example above, we simply go to Google, populate the search field with "He ### Get any page as markdown -This is a simple example of how to get any page as markdown. +This is a simple example of how to get any page as markdown, great for feeding to an LLM. ```python from dendrite import AsyncDendrite @@ -120,6 +129,56 @@ if __name__ == "__main__": asyncio.run(main()) ``` +### Get Company Data from Y Combinator + +The classic web data extraction test, made easy: + +```python +from dendrite import AsyncDendrite +import pprint +import asyncio + + +async def main(): + browser = AsyncDendrite() + + # Navigate + await browser.goto("https://www.ycombinator.com/companies") + + # Find and fill the search field with "AI agent" + await browser.fill( + "Search field", value="AI agent" + ) # Element selector cached since before + await browser.press("Enter") + + # Extract startups with natural language description + # Once created by our agent, the same script will be cached and reused + startups = await browser.extract( + "All companies. Return a list of dicts with name, location, description and url" + ) + pprint.pprint(startups, indent=2) + + +if __name__ == "__main__": + asyncio.run(main()) + +``` + +returns +``` +[ { 'description': 'Book accommodations around the world.', + 'location': 'San Francisco, CA, USA', + 'name': 'Airbnb', + 'url': 'https://www.ycombinator.com/companies/airbnb'}, + { 'description': 'Digital Analytics Platform', + 'location': 'San Francisco, CA, USA', + 'name': 'Amplitude', + 'url': 'https://www.ycombinator.com/companies/amplitude'}, +... +] } +``` + + ### Extract Data from Google Analytics Here's how to get the amount of monthly visitors from Google Analytics using the `extract` function: