Merge pull request #1945 from kili-technology/feature/lab-3849-aa-sdk-user-when-i-request-assets-it-changes-default-field

feat: asset default fields depend on project workflow version

Author: HNicolas

src/kili/domain/asset/asset.py

@@ -1,8 +1,9 @@
 """Asset domain."""
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Literal, NewType, Optional
+from typing import TYPE_CHECKING, Literal, NewType, Optional, TypedDict
 
+from kili.domain.project import WorkflowVersion
 from kili.domain.types import ListOrTuple
 
 if TYPE_CHECKING:
@@ -63,3 +64,51 @@ class AssetFilters:
     status_in: Optional[ListOrTuple[AssetStatus]] = None
     step_id_in: Optional[ListOrTuple[str]] = None
     step_status_in: Optional[ListOrTuple[StatusInStep]] = None
+
+
+class AssetWorkflowFilters(TypedDict):
+    """Asset filters relative to worklow."""
+
+    skipped: Optional[bool]
+    status_in: Optional[ListOrTuple[AssetStatus]]
+    step_name_in: Optional[ListOrTuple[str]]
+    step_status_in: Optional[ListOrTuple[StatusInStep]]
+
+
+def get_asset_default_fields(
+    project_workflow_version: WorkflowVersion,
+) -> ListOrTuple[str]:
+    """Get asset default fields for depending on the project workflow version."""
+    if project_workflow_version == "V1":
+        return (
+            "content",
+            "createdAt",
+            "externalId",
+            "id",
+            "isHoneypot",
+            "jsonMetadata",
+            "labels.author.id",
+            "labels.author.email",
+            "labels.createdAt",
+            "labels.id",
+            "labels.jsonResponse",
+            "skipped",
+            "status",
+        )
+
+    return (
+        "content",
+        "createdAt",
+        "externalId",
+        "id",
+        "isHoneypot",
+        "jsonMetadata",
+        "labels.author.id",
+        "labels.author.email",
+        "labels.createdAt",
+        "labels.id",
+        "labels.jsonResponse",
+        "skipped",
+        "currentStep.name",
+        "currentStep.status",
+    )

src/kili/domain/asset/helpers.py

@@ -1,9 +1,11 @@
 """Helpers for the asset domain."""
 
+import warnings
 from typing import Optional
 
 from kili.domain.asset import AssetExternalId, AssetId
-from kili.domain.project import ProjectId
+from kili.domain.asset.asset import AssetWorkflowFilters
+from kili.domain.project import ProjectId, WorkflowVersion
 from kili.domain.types import ListOrTuple
 from kili.exceptions import IncompatibleArgumentsError, MissingArgumentError
 
@@ -25,3 +27,40 @@ def check_asset_identifier_arguments(
         raise MissingArgumentError(
             "Either provide asset IDs or project ID with asset external IDs."
         )
+
+
+def check_asset_workflow_arguments(
+    project_workflow_version: WorkflowVersion, asset_workflow_filters: AssetWorkflowFilters
+) -> None:
+    """Check asset workflow parameters relative to the project workflow version."""
+    step_name_in = asset_workflow_filters.get("step_name_in")
+    step_status_in = asset_workflow_filters.get("step_status_in")
+    status_in = asset_workflow_filters.get("status_in")
+    skipped = asset_workflow_filters.get("skipped")
+
+    if project_workflow_version == "V2":
+        if step_status_in is not None and status_in is not None:
+            raise ValueError(
+                "Filters step_status_in and status_in both given : only use filter step_status_in for this project."
+            )
+        if step_name_in is not None and status_in is not None:
+            raise ValueError(
+                "Filters step_name_in and status_in both given : use filter step_status_in instead of status_in for this project."  # pylint: disable=line-too-long
+            )
+        if status_in is not None:
+            warnings.warn(
+                "Filter status_in given : use filters step_status_in and step_name_in instead for this project.",
+                stacklevel=1,
+            )
+        if skipped is not None:
+            warnings.warn(
+                "Filter skipped given : only use filter step_status_in with the SKIPPED step status instead for this project",  # pylint: disable=line-too-long
+                stacklevel=1,
+            )
+        return
+
+    # project workflow v1
+    if step_name_in is not None or step_status_in is not None:
+        raise ValueError(
+            "Filters step_name_in and/or step_status_in given : use filter status_in for this project."
+        )

src/kili/domain/project.py

@@ -15,6 +15,7 @@
 InputType = Literal[
     "IMAGE", "GEOSPATIAL", "PDF", "TEXT", "VIDEO", "LLM_RLHF", "LLM_INSTR_FOLLOWING", "LLM_STATIC"
 ]
+WorkflowVersion = Literal["V1", "V2"]
 
 
 @dataclass(frozen=True)

src/kili/llm/presentation/client/llm.py

@@ -2,19 +2,14 @@
 
 # pylint: disable=too-many-lines
 import warnings
-from typing import (
-    Dict,
-    List,
-    Optional,
-    Union,
-    cast,
-)
+from typing import Dict, List, Optional, Union, cast
 
 from kili_formats.types import ChatItem, ChatItemRole, Conversation
 
 from kili.adapters.kili_api_gateway.kili_api_gateway import KiliAPIGateway
 from kili.domain.asset import AssetExternalId, AssetFilters, AssetId, AssetStatus
 from kili.domain.asset.asset import StatusInStep
+from kili.domain.asset.helpers import check_asset_workflow_arguments
 from kili.domain.label import LabelType
 from kili.domain.llm import (
     AzureOpenAICredentials,
@@ -31,7 +26,9 @@
 )
 from kili.domain.project import ProjectId
 from kili.llm.services.export import export
-from kili.presentation.client.helpers.filter_conversion import convert_step_in_to_step_id_in_filter
+from kili.presentation.client.helpers.filter_conversion import (
+    extract_step_ids_from_project_steps,
+)
 from kili.services.export.exceptions import NoCompatibleJobError
 from kili.use_cases.asset.utils import AssetUseCasesUtils
 from kili.use_cases.project.project import ProjectUseCases
@@ -128,15 +125,26 @@ def export(
 
         if status_in is not None or step_name_in is not None or step_status_in is not None:
             project_use_cases = ProjectUseCases(self.kili_api_gateway)
-            step_id_in = convert_step_in_to_step_id_in_filter(
-                project_steps=project_use_cases.get_project_steps(project_id),
-                asset_filter_kwargs={
+            (
+                project_steps,
+                project_workflow_version,
+            ) = project_use_cases.get_project_steps_and_version(project_id)
+            check_asset_workflow_arguments(
+                project_workflow_version=project_workflow_version,
+                asset_workflow_filters={
+                    "skipped": None,
+                    "status_in": status_in,
                     "step_name_in": step_name_in,
                     "step_status_in": step_status_in,
-                    "status_in": status_in,
                 },
             )
 
+            if project_workflow_version == "V2" and step_name_in is not None:
+                step_id_in = extract_step_ids_from_project_steps(
+                    project_steps=project_steps,
+                    step_name_in=step_name_in,
+                )
+
         asset_filter = AssetFilters(
             project_id=ProjectId(project_id),
             asset_id_in=resolved_asset_ids,

src/kili/presentation/client/asset.py

@@ -23,7 +23,9 @@
     AssetId,
     AssetStatus,
     StatusInStep,
+    get_asset_default_fields,
 )
+from kili.domain.asset.helpers import check_asset_workflow_arguments
 from kili.domain.issue import IssueStatus, IssueType
 from kili.domain.label import LabelType
 from kili.domain.project import ProjectId
@@ -32,7 +34,7 @@
     disable_tqdm_if_as_generator,
 )
 from kili.presentation.client.helpers.filter_conversion import (
-    convert_step_in_to_step_id_in_filter,
+    extract_step_ids_from_project_steps,
 )
 from kili.use_cases.asset import AssetUseCases
 from kili.use_cases.project.project import ProjectUseCases
@@ -55,21 +57,7 @@ def assets(
         project_id: str,
         asset_id: Optional[str] = None,
         skip: int = 0,
-        fields: ListOrTuple[str] = (
-            "content",
-            "createdAt",
-            "externalId",
-            "id",
-            "isHoneypot",
-            "jsonMetadata",
-            "labels.author.id",
-            "labels.author.email",
-            "labels.createdAt",
-            "labels.id",
-            "labels.jsonResponse",
-            "skipped",
-            "status",
-        ),
+        fields: Optional[ListOrTuple[str]] = None,
         asset_id_in: Optional[List[str]] = None,
         asset_id_not_in: Optional[List[str]] = None,
         consensus_mark_gt: Optional[float] = None,
@@ -135,21 +123,7 @@ def assets(
         project_id: str,
         asset_id: Optional[str] = None,
         skip: int = 0,
-        fields: ListOrTuple[str] = (
-            "content",
-            "createdAt",
-            "externalId",
-            "id",
-            "isHoneypot",
-            "jsonMetadata",
-            "labels.author.id",
-            "labels.author.email",
-            "labels.createdAt",
-            "labels.id",
-            "labels.jsonResponse",
-            "skipped",
-            "status",
-        ),
+        fields: Optional[ListOrTuple[str]] = None,
         asset_id_in: Optional[List[str]] = None,
         asset_id_not_in: Optional[List[str]] = None,
         consensus_mark_gt: Optional[float] = None,
@@ -215,21 +189,7 @@ def assets(
         project_id: str,
         asset_id: Optional[str] = None,
         skip: int = 0,
-        fields: ListOrTuple[str] = (
-            "content",
-            "createdAt",
-            "externalId",
-            "id",
-            "isHoneypot",
-            "jsonMetadata",
-            "labels.author.id",
-            "labels.author.email",
-            "labels.createdAt",
-            "labels.id",
-            "labels.jsonResponse",
-            "skipped",
-            "status",
-        ),
+        fields: Optional[ListOrTuple[str]] = None,
         asset_id_in: Optional[List[str]] = None,
         asset_id_not_in: Optional[List[str]] = None,
         consensus_mark_gt: Optional[float] = None,
@@ -453,31 +413,44 @@ def assets(
 
         disable_tqdm = disable_tqdm_if_as_generator(as_generator, disable_tqdm)
 
+        project_use_cases = ProjectUseCases(self.kili_api_gateway)
+        project_steps, project_workflow_version = project_use_cases.get_project_steps_and_version(
+            project_id
+        )
+        if fields is None:
+            fields = get_asset_default_fields(project_workflow_version=project_workflow_version)
+        elif project_workflow_version == "V1":
+            for invalid_field in filter(lambda f: f.startswith("currentStep."), fields):
+                warnings.warn(
+                    f"Field {invalid_field} requested : request 'status' field instead for this project",
+                    stacklevel=1,
+                )
+        elif "status" in fields:
+            warnings.warn(
+                "Field status requested : request 'currentStep.name' and 'currentStep.status' fields instead for this project",
+                stacklevel=1,
+            )
+
         step_id_in = None
         if (
             step_name_in is not None
             or step_status_in is not None
             or status_in is not None
             or skipped is not None
         ):
-            project_use_cases = ProjectUseCases(self.kili_api_gateway)
-            project_steps = project_use_cases.get_project_steps(project_id)
-
-            if step_name_in is not None or step_status_in is not None or status_in is not None:
-                step_id_in = convert_step_in_to_step_id_in_filter(
+            check_asset_workflow_arguments(
+                project_workflow_version=project_workflow_version,
+                asset_workflow_filters={
+                    "skipped": skipped,
+                    "status_in": status_in,
+                    "step_name_in": step_name_in,
+                    "step_status_in": step_status_in,
+                },
+            )
+            if project_workflow_version == "V2" and step_name_in is not None:
+                step_id_in = extract_step_ids_from_project_steps(
                     project_steps=project_steps,
-                    fields=fields,
-                    asset_filter_kwargs={
-                        "step_name_in": step_name_in,
-                        "step_status_in": step_status_in,
-                        "status_in": status_in,
-                        "skipped": skipped,
-                    },
-                )
-            elif skipped is not None and len(project_steps) != 0:
-                warnings.warn(
-                    "Filter skipped given : only use filter step_status_in with the SKIPPED step status instead for this project",
-                    stacklevel=1,
+                    step_name_in=step_name_in,
                 )
 
         asset_use_cases = AssetUseCases(self.kili_api_gateway)
@@ -722,18 +695,28 @@ def count_assets(
                 )
 
         step_id_in = None
-
         if status_in is not None or step_name_in is not None or step_status_in is not None:
             project_use_cases = ProjectUseCases(self.kili_api_gateway)
-            step_id_in = convert_step_in_to_step_id_in_filter(
-                project_steps=project_use_cases.get_project_steps(project_id),
-                asset_filter_kwargs={
+            (
+                project_steps,
+                project_workflow_version,
+            ) = project_use_cases.get_project_steps_and_version(project_id)
+            check_asset_workflow_arguments(
+                project_workflow_version=project_workflow_version,
+                asset_workflow_filters={
+                    "skipped": skipped,
                     "step_name_in": step_name_in,
                     "step_status_in": step_status_in,
                     "status_in": status_in,
                 },
             )
 
+            if project_workflow_version == "V2" and step_name_in is not None:
+                step_id_in = extract_step_ids_from_project_steps(
+                    project_steps=project_steps,
+                    step_name_in=step_name_in,
+                )
+
         filters = AssetFilters(
             project_id=ProjectId(project_id),
             asset_id=AssetId(asset_id) if asset_id else None,