Webhook jobs (#3379)

* webhook jobs

* minor

* very minor

* create_webhook(job_id=job_id, ...)

* fix tests

* Apply suggestions from code review

Co-authored-by: Lucain <lucain@huggingface.co>

---------

Co-authored-by: Lucain <lucain@huggingface.co>

Author: lhoestq

docs/source/en/guides/jobs.md

@@ -410,3 +410,24 @@ Manage scheduled jobs using [`list_scheduled_jobs`], [`inspect_scheduled_job`],
 >>> from huggingface_hub import delete_scheduled_job
 >>> delete_scheduled_job(scheduled_job_id)
 ```
+
+### Trigger Jobs with webhooks
+
+Webhooks allow you to listen for new changes on specific repos or to all repos belonging to particular set of users/organizations (not just your repos, but any repo).
+
+Use [`create_webhook`] to create a webhook that triggers a Job when a change happens in a Hugging Face repository:
+
+```python
+from huggingface_hub import create_webhook
+
+# Example: Creating a webhook that triggers a Job
+webhook = create_webhook(
+    job_id=job_id,
+    watched=[{"type": "user", "name": "your-username"}, {"type": "org", "name": "your-org-name"}],
+    domains=["repo", "discussion"],
+    secret="your-secret"
+)
+```
+
+The webhook triggers the Job with the webhook payload in the environment variable `WEBHOOK_PAYLOAD`.
+You can find more information on webhooks in the [Webhooks documentation](./webhooks).

docs/source/en/guides/webhooks.md

@@ -28,6 +28,24 @@ webhook = create_webhook(
 )
 ```
 
+A webhook can also trigger a Job to run on Hugging face infrastructure instead of sending the payload to an URL.
+In this case you need to pass the ID of a source Job.
+
+```python
+from huggingface_hub import create_webhook
+
+# Example: Creating a webhook that triggers a Job
+webhook = create_webhook(
+    job_id=job_id,
+    watched=[{"type": "user", "name": "your-username"}, {"type": "org", "name": "your-org-name"}],
+    domains=["repo", "discussion"],
+    secret="your-secret"
+)
+```
+
+The webhook triggers the Job with the webhook payload in the environment variable `WEBHOOK_PAYLOAD`.
+For more information on Hugging Face Jobs, available hardware (CPU, GPU) and UV scripts, see the [Jobs documentation](./jobs).
+
 ### Listing Webhooks
 
 To see all the webhooks you have configured, you can list them with [`list_webhooks`]. This is useful to review their IDs, URLs, and statuses.

src/huggingface_hub/hf_api.py

@@ -65,7 +65,7 @@
     _warn_on_overwriting_operations,
 )
 from ._inference_endpoints import InferenceEndpoint, InferenceEndpointType
-from ._jobs_api import JobInfo, ScheduledJobInfo, _create_job_spec
+from ._jobs_api import JobInfo, JobSpec, ScheduledJobInfo, _create_job_spec
 from ._space_api import SpaceHardware, SpaceRuntime, SpaceStorage, SpaceVariable
 from ._upload_large_folder import upload_large_folder_internal
 from .community import (
@@ -503,11 +503,15 @@ class WebhookWatchedItem:
 class WebhookInfo:
     """Data structure containing information about a webhook.
 
+    One of `url` or `job` is specified, but not both.
+
     Attributes:
         id (`str`):
             ID of the webhook.
-        url (`str`):
+        url (`str`, *optional*):
             URL of the webhook.
+        job (`JobSpec`, *optional*):
+            Specifications of the Job to trigger.
         watched (`List[WebhookWatchedItem]`):
             List of items watched by the webhook, see [`WebhookWatchedItem`].
         domains (`List[WEBHOOK_DOMAIN_T]`):
@@ -519,7 +523,8 @@ class WebhookInfo:
     """
 
     id: str
-    url: str
+    url: Optional[str]
+    job: Optional[JobSpec]
     watched: List[WebhookWatchedItem]
     domains: List[constants.WEBHOOK_DOMAIN_T]
     secret: Optional[str]
@@ -9035,6 +9040,7 @@ def get_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = None)
             >>> print(webhook)
             WebhookInfo(
                 id="654bbbc16f2ec14d77f109cc",
+                job=None,
                 watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")],
                 url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548",
                 secret="my-secret",
@@ -9054,7 +9060,8 @@ def get_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = None)
 
         webhook = WebhookInfo(
             id=webhook_data["id"],
-            url=webhook_data["url"],
+            url=webhook_data.get("url"),
+            job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None,
             watched=watched_items,
             domains=webhook_data["domains"],
             secret=webhook_data.get("secret"),
@@ -9104,7 +9111,8 @@ def list_webhooks(self, *, token: Union[bool, str, None] = None) -> List[Webhook
         return [
             WebhookInfo(
                 id=webhook["id"],
-                url=webhook["url"],
+                url=webhook.get("url"),
+                job=JobSpec(**webhook["job"]) if webhook.get("job") else None,
                 watched=[WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook["watched"]],
                 domains=webhook["domains"],
                 secret=webhook.get("secret"),
@@ -9117,17 +9125,24 @@ def list_webhooks(self, *, token: Union[bool, str, None] = None) -> List[Webhook
     def create_webhook(
         self,
         *,
-        url: str,
+        url: Optional[str] = None,
+        job_id: Optional[str] = None,
         watched: List[Union[Dict, WebhookWatchedItem]],
         domains: Optional[List[constants.WEBHOOK_DOMAIN_T]] = None,
         secret: Optional[str] = None,
         token: Union[bool, str, None] = None,
     ) -> WebhookInfo:
         """Create a new webhook.
 
+        The webhook can either send a payload to a URL, or trigger a Job to run on Hugging Face infrastructure.
+        This function should be called with one of `url` or `job_id`, but not both.
+
         Args:
             url (`str`):
                 URL to send the payload to.
+            job_id (`str`):
+                ID of the source Job to trigger with the webhook payload in the environment variable WEBHOOK_PAYLOAD.
+                Additional environment variables are available for convenience: WEBHOOK_REPO_ID, WEBHOOK_REPO_TYPE and WEBHOOK_SECRET.
             watched (`List[WebhookWatchedItem]`):
                 List of [`WebhookWatchedItem`] to be watched by the webhook. It can be users, orgs, models, datasets or spaces.
                 Watched items can also be provided as plain dictionaries.
@@ -9145,6 +9160,8 @@ def create_webhook(
                 Info about the newly created webhook.
 
         Example:
+
+            Create a webhook that sends a payload to a URL
             ```python
             >>> from huggingface_hub import create_webhook
             >>> payload = create_webhook(
@@ -9157,6 +9174,43 @@ def create_webhook(
             WebhookInfo(
                 id="654bbbc16f2ec14d77f109cc",
                 url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548",
+                job=None,
+                watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")],
+                domains=["repo", "discussion"],
+                secret="my-secret",
+                disabled=False,
+            )
+            ```
+
+            Run a Job and then create a webhook that triggers this Job
+            ```python
+            >>> from huggingface_hub import create_webhook, run_job
+            >>> job = run_job(
+            ...     image="ubuntu",
+            ...     command=["bash", "-c", r"echo An event occured in $WEBHOOK_REPO_ID: $WEBHOOK_PAYLOAD"],
+            ... )
+            >>> payload = create_webhook(
+            ...     watched=[{"type": "user", "name": "julien-c"}, {"type": "org", "name": "HuggingFaceH4"}],
+            ...     job_id=job.id,
+            ...     domains=["repo", "discussion"],
+            ...     secret="my-secret",
+            ... )
+            >>> print(payload)
+            WebhookInfo(
+                id="654bbbc16f2ec14d77f109cc",
+                url=None,
+                job=JobSpec(
+                    docker_image='ubuntu',
+                    space_id=None,
+                    command=['bash', '-c', 'echo An event occured in $WEBHOOK_REPO_ID: $WEBHOOK_PAYLOAD'],
+                    arguments=[],
+                    environment={},
+                    secrets=[],
+                    flavor='cpu-basic',
+                    timeout=None,
+                    tags=None,
+                    arch=None
+                ),
                 watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")],
                 domains=["repo", "discussion"],
                 secret="my-secret",
@@ -9166,9 +9220,19 @@ def create_webhook(
         """
         watched_dicts = [asdict(item) if isinstance(item, WebhookWatchedItem) else item for item in watched]
 
+        post_webhooks_json = {"watched": watched_dicts, "domains": domains, "secret": secret}
+        if url is not None and job_id is not None:
+            raise ValueError("Set `url` or `job_id` but not both.")
+        elif url is not None:
+            post_webhooks_json["url"] = url
+        elif job_id is not None:
+            post_webhooks_json["jobSourceId"] = job_id
+        else:
+            raise ValueError("Missing argument for webhook: `url` or `job_id`.")
+
         response = get_session().post(
             f"{constants.ENDPOINT}/api/settings/webhooks",
-            json={"watched": watched_dicts, "url": url, "domains": domains, "secret": secret},
+            json=post_webhooks_json,
             headers=self._build_hf_headers(token=token),
         )
         hf_raise_for_status(response)
@@ -9177,7 +9241,8 @@ def create_webhook(
 
         webhook = WebhookInfo(
             id=webhook_data["id"],
-            url=webhook_data["url"],
+            url=webhook_data.get("url"),
+            job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None,
             watched=watched_items,
             domains=webhook_data["domains"],
             secret=webhook_data.get("secret"),
@@ -9233,6 +9298,7 @@ def update_webhook(
             >>> print(updated_payload)
             WebhookInfo(
                 id="654bbbc16f2ec14d77f109cc",
+                job=None,
                 url="https://new.webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548",
                 watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")],
                 domains=["repo"],
@@ -9256,7 +9322,8 @@ def update_webhook(
 
         webhook = WebhookInfo(
             id=webhook_data["id"],
-            url=webhook_data["url"],
+            url=webhook_data.get("url"),
+            job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None,
             watched=watched_items,
             domains=webhook_data["domains"],
             secret=webhook_data.get("secret"),
@@ -9288,6 +9355,7 @@ def enable_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = Non
             >>> enabled_webhook
             WebhookInfo(
                 id="654bbbc16f2ec14d77f109cc",
+                job=None,
                 url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548",
                 watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")],
                 domains=["repo", "discussion"],
@@ -9307,7 +9375,8 @@ def enable_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = Non
 
         webhook = WebhookInfo(
             id=webhook_data["id"],
-            url=webhook_data["url"],
+            url=webhook_data.get("url"),
+            job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None,
             watched=watched_items,
             domains=webhook_data["domains"],
             secret=webhook_data.get("secret"),
@@ -9340,6 +9409,7 @@ def disable_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = No
             WebhookInfo(
                 id="654bbbc16f2ec14d77f109cc",
                 url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548",
+                jon=None,
                 watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")],
                 domains=["repo", "discussion"],
                 secret="my-secret",
@@ -9358,7 +9428,8 @@ def disable_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = No
 
         webhook = WebhookInfo(
             id=webhook_data["id"],
-            url=webhook_data["url"],
+            url=webhook_data.get("url"),
+            job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None,
             watched=watched_items,
             domains=webhook_data["domains"],
             secret=webhook_data.get("secret"),