refactor!: Adapt to the Crawlee v1.0 (#470)

### Description

- Integration of the Crawlee v1 changes, mostly new storages & storage
clients (introduced in
apify/crawlee-python#1194).

### Issues

- Closes: #469
- Closes: #540 

### Testing

- The current test set covers the changes.

---------

Co-authored-by: Josef Prochazka <josef.prochazka@apify.com>

Author: vdusek

.github/workflows/run_code_checks.yaml

@@ -36,7 +36,6 @@ jobs:
 
   integration_tests:
     name: Integration tests
-    needs: [lint_check, type_check, unit_tests]
     uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main
     secrets: inherit
     with:

Makefile

@@ -26,13 +26,13 @@ type-check:
 	uv run mypy
 
 unit-tests:
-	uv run pytest --numprocesses=auto --verbose --cov=src/apify tests/unit
+	uv run pytest --numprocesses=auto -vv --cov=src/apify tests/unit
 
 unit-tests-cov:
-	uv run pytest --numprocesses=auto --verbose --cov=src/apify --cov-report=html tests/unit
+	uv run pytest --numprocesses=auto -vv --cov=src/apify --cov-report=html tests/unit
 
 integration-tests:
-	uv run pytest --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) --verbose tests/integration
+	uv run pytest --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) -vv tests/integration
 
 format:
 	uv run ruff check --fix

docs/03_concepts/code/03_dataset_exports.py

@@ -11,14 +11,14 @@ async def main() -> None:
         await dataset.export_to(
             content_type='csv',
             key='data.csv',
-            to_key_value_store_name='my-cool-key-value-store',
+            to_kvs_name='my-cool-key-value-store',
         )
 
         # Export the data as JSON
         await dataset.export_to(
             content_type='json',
             key='data.json',
-            to_key_value_store_name='my-cool-key-value-store',
+            to_kvs_name='my-cool-key-value-store',
         )
 
         # Print the exported records

docs/03_concepts/code/conditional_actor_charge.py

@@ -6,8 +6,8 @@ async def main() -> None:
         # Check the dataset because there might already be items
         # if the run migrated or was restarted
         default_dataset = await Actor.open_dataset()
-        dataset_info = await default_dataset.get_info()
-        charged_items = dataset_info.item_count if dataset_info else 0
+        metadata = await default_dataset.get_metadata()
+        charged_items = metadata.item_count
 
         # highlight-start
         if Actor.get_charging_manager().get_pricing_info().is_pay_per_event:

docs/04_upgrading/upgrading_to_v2.md

@@ -3,7 +3,7 @@ id: upgrading-to-v2
 title: Upgrading to v2
 ---
 
-This page summarizes most of the breaking changes between Apify Python SDK v1.x and v2.0.
+This page summarizes the breaking changes between Apify Python SDK v1.x and v2.0.
 
 ## Python version support
 
@@ -12,7 +12,7 @@ Support for Python 3.8 has been dropped. The Apify Python SDK v2.x now requires
 ## Storages
 
 - The SDK now uses [crawlee](https://github.com/apify/crawlee-python) for local storage emulation. This change should not affect intended usage (working with `Dataset`, `KeyValueStore` and `RequestQueue` classes from the `apify.storages` module or using the shortcuts exposed by the `Actor` class) in any way.
-- There is a difference in the `RequestQueue.add_request` method: it accepts an `apify.Request` object instead of a free-form dictionary. 
+- There is a difference in the `RequestQueue.add_request` method: it accepts an `apify.Request` object instead of a free-form dictionary.
     - A quick way to migrate from dict-based arguments is to wrap it with a `Request.model_validate()` call.
     - The preferred way is using the `Request.from_url` helper which prefills the `unique_key` and `id` attributes, or instantiating it directly, e.g., `Request(url='https://example.tld', ...)`.
     - For simple use cases, `add_request` also accepts plain strings that contain an URL, e.g. `queue.add_request('https://example.tld')`.

docs/04_upgrading/upgrading_to_v3.md

@@ -0,0 +1,18 @@
+---
+id: upgrading-to-v2
+title: Upgrading to v2
+---
+
+This page summarizes the breaking changes between Apify Python SDK v2.x and v3.0.
+
+## Python version support
+
+Support for Python 3.9 has been dropped. The Apify Python SDK v3.x now requires Python 3.10 or later. Make sure your environment is running a compatible version before upgrading.
+
+## Storages
+
+<!-- TODO -->
+
+## Storage clients
+
+<!-- TODO -->

pyproject.toml

@@ -36,7 +36,8 @@ keywords = [
 dependencies = [
     "apify-client<2.0.0",
     "apify-shared<2.0.0",
-    "crawlee~=0.6.0",
+    "crawlee@git+https://github.com/apify/crawlee-python.git@master",
+    "cachetools>=5.5.0",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
     # TODO: ensure compatibility with the latest version of lazy-object-proxy
@@ -77,6 +78,7 @@ dev = [
     "pytest~=8.4.0",
     "ruff~=0.12.0",
     "setuptools", # setuptools are used by pytest but not explicitly required
+    "types-cachetools>=6.0.0.20250525",
     "uvicorn[standard]",
     "werkzeug~=3.1.3", # Werkzeug is used by httpserver
     "yarl~=1.20.0", # yarl is used by crawlee
@@ -85,6 +87,9 @@ dev = [
 [tool.hatch.build.targets.wheel]
 packages = ["src/apify"]
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.ruff]
 line-length = 120
 include = ["src/**/*.py", "tests/**/*.py", "docs/**/*.py", "website/**/*.py"]

src/apify/_actor.py

@@ -30,11 +30,11 @@
 from apify._consts import EVENT_LISTENERS_TIMEOUT
 from apify._crypto import decrypt_input_secrets, load_private_key
 from apify._models import ActorRun
-from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
 from apify._proxy_configuration import ProxyConfiguration
 from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython
-from apify.apify_storage_client import ApifyStorageClient
+from apify.events import ApifyEventManager, EventManager, LocalEventManager
 from apify.log import _configure_logging, logger
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:
@@ -126,11 +126,11 @@ def __init__(
 
         # Create an instance of the cloud storage client, the local storage client is obtained
         # from the service locator.
-        self._cloud_storage_client = ApifyStorageClient.from_config(config=self._configuration)
+        self._cloud_storage_client = ApifyStorageClient()
 
         # Set the event manager based on whether the Actor is running on the platform or locally.
         self._event_manager = (
-            PlatformEventManager(
+            ApifyEventManager(
                 config=self._configuration,
                 persist_state_interval=self._configuration.persist_state_interval,
             )

src/apify/_configuration.py

@@ -140,6 +140,39 @@ class Configuration(CrawleeConfiguration):
         ),
     ] = None
 
+    default_dataset_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_dataset_id',
+                'apify_default_dataset_id',
+            ),
+            description='Default dataset ID used by the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
+    default_key_value_store_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_key_value_store_id',
+                'apify_default_key_value_store_id',
+            ),
+            description='Default key-value store ID for the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
+    default_request_queue_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_request_queue_id',
+                'apify_default_request_queue_id',
+            ),
+            description='Default request queue ID for the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
     disable_outdated_warning: Annotated[
         bool,
         Field(

src/apify/_proxy_configuration.py

@@ -20,7 +20,8 @@
 
 if TYPE_CHECKING:
     from apify_client import ApifyClientAsync
-    from crawlee import Request
+
+    from apify import Request
 
 APIFY_PROXY_VALUE_REGEX = re.compile(r'^[\w._~]+$')
 COUNTRY_CODE_REGEX = re.compile(r'^[A-Z]{2}$')