chore(docs): add annotation queues and evaluator playground updates to roadmap

api/entrypoints/routers.py

@@ -144,7 +144,6 @@
 
 from oss.src.utils.env import env
 from entrypoints.worker_evaluations import evaluations_worker
-from entrypoints.worker_webhooks import webhooks_worker
 import oss.src.core.evaluations.tasks.live  # noqa: F401
 import oss.src.core.evaluations.tasks.legacy  # noqa: F401
 import oss.src.core.evaluations.tasks.batch  # noqa: F401
@@ -277,7 +276,6 @@ async def lifespan(*args, **kwargs):
 webhooks_service = WebhooksService(
     webhooks_dao=webhooks_dao,
     vault_service=vault_service,
-    webhooks_worker=webhooks_worker,
 )
 
 

api/oss/src/apis/fastapi/webhooks/models.py

@@ -1,4 +1,5 @@
-from typing import List, Optional
+from typing import List, Optional, Union
+from uuid import UUID
 
 from pydantic import BaseModel
 
@@ -24,6 +25,13 @@ class WebhookSubscriptionEditRequest(BaseModel):
     subscription: WebhookSubscriptionEdit
 
 
+class WebhookSubscriptionTestRequest(BaseModel):
+    subscription_id: Optional[UUID] = None
+    subscription: Optional[
+        Union[WebhookSubscriptionEdit, WebhookSubscriptionCreate]
+    ] = None
+
+
 class WebhookSubscriptionQueryRequest(BaseModel):
     subscription: Optional[WebhookSubscriptionQuery] = None
 

api/oss/src/apis/fastapi/webhooks/router.py

@@ -12,15 +12,14 @@
 )
 from oss.src.utils.crypting import decrypt, encrypt
 from oss.src.core.webhooks.service import WebhooksService
-from oss.src.core.webhooks.types import WebhookSubscription
+from oss.src.core.webhooks.types import WebhookSubscription, WebhookSubscriptionEdit
 from oss.src.core.webhooks.exceptions import (
     WebhookAuthorizationSecretRequiredError,
     WebhookSubscriptionNotFoundError,
-    WebhookTestDeliveryTimeoutError,
-    WebhookTestEventPublishFailedError,
 )
 from oss.src.apis.fastapi.webhooks.models import (
     WebhookSubscriptionCreateRequest,
+    WebhookSubscriptionTestRequest,
     WebhookSubscriptionEditRequest,
     WebhookSubscriptionQueryRequest,
     WebhookSubscriptionResponse,
@@ -50,7 +49,7 @@ def __init__(
         # --- WEBHOOK SUBSCRIPTIONS ------------------------------------------ #
 
         self.router.add_api_route(
-            "/",
+            "/subscriptions/",
             self.create_subscription,
             methods=["POST"],
             operation_id="create_webhook_subscription",
@@ -59,7 +58,16 @@ def __init__(
             status_code=status.HTTP_200_OK,
         )
         self.router.add_api_route(
-            "/{subscription_id}",
+            "/subscriptions/test",
+            self.test_subscription,
+            methods=["POST"],
+            operation_id="test_webhook_subscription",
+            response_model=WebhookDeliveryResponse,
+            response_model_exclude_none=True,
+            status_code=status.HTTP_200_OK,
+        )
+        self.router.add_api_route(
+            "/subscriptions/{subscription_id}",
             self.fetch_subscription,
             methods=["GET"],
             operation_id="fetch_webhook_subscription",
@@ -68,7 +76,7 @@ def __init__(
             status_code=status.HTTP_200_OK,
         )
         self.router.add_api_route(
-            "/{subscription_id}",
+            "/subscriptions/{subscription_id}",
             self.edit_subscription,
             methods=["PUT"],
             operation_id="edit_webhook_subscription",
@@ -77,14 +85,14 @@ def __init__(
             status_code=status.HTTP_200_OK,
         )
         self.router.add_api_route(
-            "/{subscription_id}",
+            "/subscriptions/{subscription_id}",
             self.delete_subscription,
             methods=["DELETE"],
             operation_id="delete_webhook_subscription",
             status_code=status.HTTP_204_NO_CONTENT,
         )
         self.router.add_api_route(
-            "/query",
+            "/subscriptions/query",
             self.query_subscriptions,
             methods=["POST"],
             operation_id="query_webhook_subscriptions",
@@ -123,18 +131,6 @@ def __init__(
             status_code=status.HTTP_200_OK,
         )
 
-        # --- WEBHOOK TEST --------------------------------------------------- #
-
-        self.router.add_api_route(
-            "/test/{subscription_id}",
-            self.test_webhook,
-            methods=["POST"],
-            operation_id="test_webhook",
-            response_model=WebhookDeliveryResponse,
-            response_model_exclude_none=True,
-            status_code=status.HTTP_200_OK,
-        )
-
     # --- WEBHOOK SUBSCRIPTIONS ---------------------------------------------- #
 
     @intercept_exceptions()
@@ -467,12 +463,11 @@ async def query_deliveries(
 
     # --- WEBHOOK TESTS ------------------------------------------------------ #
 
-    @intercept_exceptions()
-    async def test_webhook(
+    async def _test_subscription_impl(
         self,
-        request: Request,
         *,
-        subscription_id: UUID,
+        request: Request,
+        body: WebhookSubscriptionTestRequest,
     ) -> WebhookDeliveryResponse:
         if is_ee():
             has_permission = await check_action_access(
@@ -483,52 +478,60 @@ async def test_webhook(
             if not has_permission:
                 raise FORBIDDEN_EXCEPTION  # type: ignore
 
-        try:
-            delivery = await self.webhooks_service.test_webhook(
+        if (body.subscription_id is None) == (body.subscription is None):
+            raise HTTPException(
+                status_code=400,
+                detail="Provide exactly one of subscription_id or subscription",
+            )
+
+        subscription = body.subscription
+
+        if subscription is None:
+            assert body.subscription_id is not None
+            existing = await self.webhooks_service.fetch_subscription(
                 project_id=UUID(request.state.project_id),
-                #
-                subscription_id=subscription_id,
+                subscription_id=body.subscription_id,
             )
 
-            if delivery.status and delivery.status.message == "success":
-                await invalidate_cache(
-                    namespace="webhooks",
-                    project_id=str(request.state.project_id),
-                    key=f"subscription:{subscription_id}",
-                )
-                await invalidate_cache(
-                    namespace="webhooks",
-                    project_id=str(request.state.project_id),
-                    key="subscriptions",
+            if not existing:
+                raise HTTPException(
+                    status_code=404, detail="Webhook subscription not found"
                 )
 
-            return WebhookDeliveryResponse(
-                count=1 if delivery else 0,
-                delivery=delivery,
+            subscription = WebhookSubscriptionEdit(
+                id=existing.id,
+                name=existing.name,
+                description=existing.description,
+                data=existing.data,
+                secret=existing.secret,
             )
+
+        assert subscription is not None
+
+        try:
+            delivery = await self.webhooks_service.test_subscription(
+                project_id=UUID(request.state.project_id),
+                user_id=UUID(str(request.state.user_id)),
+                subscription=subscription,
+            )
+        except WebhookAuthorizationSecretRequiredError as e:
+            raise HTTPException(status_code=400, detail=e.message) from e
         except WebhookSubscriptionNotFoundError as e:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail=str(e),
-            ) from e
-        except WebhookTestEventPublishFailedError as e:
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail={
-                    "code": "WEBHOOK_TEST_EVENT_PUBLISH_FAILED",
-                    "message": e.message,
-                    "event_id": e.event_id,
-                    "subscription_id": e.subscription_id,
-                },
-            ) from e
-        except WebhookTestDeliveryTimeoutError as e:
-            raise HTTPException(
-                status_code=status.HTTP_504_GATEWAY_TIMEOUT,
-                detail={
-                    "code": "WEBHOOK_TEST_DELIVERY_TIMEOUT",
-                    "message": e.message,
-                    "event_id": e.event_id,
-                    "subscription_id": e.subscription_id,
-                    "attempts": e.attempts,
-                },
-            ) from e
+            raise HTTPException(status_code=404, detail=str(e)) from e
+
+        return WebhookDeliveryResponse(
+            count=1 if delivery else 0,
+            delivery=delivery,
+        )
+
+    @intercept_exceptions()
+    async def test_subscription(
+        self,
+        request: Request,
+        *,
+        body: WebhookSubscriptionTestRequest,
+    ) -> WebhookDeliveryResponse:
+        return await self._test_subscription_impl(
+            request=request,
+            body=body,
+        )