diff --git a/src/viam/components/camera/camera.py b/src/viam/components/camera/camera.py index 9faeae9d5..6174f873d 100644 --- a/src/viam/components/camera/camera.py +++ b/src/viam/components/camera/camera.py @@ -63,7 +63,7 @@ async def get_image( ... @abc.abstractmethod - async def get_images(self, *, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: + async def get_images(self, *, filter_source_names: Optional[List[str]] = None, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: """Get simultaneous images from different imagers, along with associated metadata. This should not be used for getting a time series of images from the same imager. diff --git a/src/viam/components/camera/client.py b/src/viam/components/camera/client.py index 45feacf32..493d219ed 100644 --- a/src/viam/components/camera/client.py +++ b/src/viam/components/camera/client.py @@ -41,20 +41,26 @@ async def get_image( md = kwargs.get("metadata", self.Metadata()).proto request = GetImageRequest(name=self.name, mime_type=mime_type, extra=dict_to_struct(extra)) response: GetImageResponse = await self.client.GetImage(request, timeout=timeout, metadata=md) - return ViamImage(response.image, CameraMimeType.from_string(response.mime_type)) + return ViamImage(response.image, response.mime_type) async def get_images( self, *, + filter_source_names: Optional[List[str]] = None, + extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs, ) -> Tuple[List[NamedImage], ResponseMetadata]: md = kwargs.get("metadata", self.Metadata()).proto - request = GetImagesRequest(name=self.name) + request = GetImagesRequest(name=self.name, extra=dict_to_struct(extra), filter_source_names=filter_source_names) response: GetImagesResponse = await self.client.GetImages(request, timeout=timeout, metadata=md) imgs = [] for img_data in response.images: - mime_type = CameraMimeType.from_proto(img_data.format) + if img_data.mime_type: + mime_type = img_data.mime_type + else: + # TODO(RSDK-11728): remove this once we deleted the format field + mime_type = str(CameraMimeType.from_proto(img_data.format)) img = NamedImage(img_data.source_name, img_data.image, mime_type) imgs.append(img) resp_metadata: ResponseMetadata = response.response_metadata diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 936271491..7108bb437 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -6,6 +6,7 @@ from viam.proto.common import DoCommandRequest, DoCommandResponse, GetGeometriesRequest, GetGeometriesResponse from viam.proto.component.camera import ( CameraServiceBase, + Format, GetImageRequest, GetImageResponse, GetImagesRequest, @@ -19,6 +20,7 @@ ) from viam.resource.rpc_service_base import ResourceRPCServiceBase from viam.utils import dict_to_struct, struct_to_dict +from viam.media.video import CameraMimeType from . import Camera @@ -48,12 +50,23 @@ async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) - camera = self.get_resource(name) timeout = stream.deadline.time_remaining() if stream.deadline else None - images, metadata = await camera.get_images(timeout=timeout, metadata=stream.metadata) + images, metadata = await camera.get_images( + timeout=timeout, + metadata=stream.metadata, + extra=struct_to_dict(request.extra), + filter_source_names=list(request.filter_source_names), + ) img_bytes_lst = [] for img in images: - fmt = img.mime_type.to_proto() + try: + mime_type = CameraMimeType.from_string(img.mime_type) # this can ValueError if the mime_type is not a CameraMimeType + fmt = mime_type.to_proto() + except ValueError: + # TODO(RSDK-11728): remove this once we deleted the format field + fmt = Format.FORMAT_UNSPECIFIED + img_bytes = img.data - img_bytes_lst.append(Image(source_name=name, format=fmt, image=img_bytes)) + img_bytes_lst.append(Image(source_name=name, mime_type=img.mime_type, format=fmt, image=img_bytes)) response = GetImagesResponse(images=img_bytes_lst, response_metadata=metadata) await stream.send_message(response) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 3b3f7fcc2..e016e5136 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -1,7 +1,6 @@ from array import array from enum import Enum from typing import List, Optional, Tuple - from typing_extensions import Self from viam.errors import NotSupportedError @@ -28,7 +27,10 @@ def from_string(cls, value: str) -> Self: Self: The mimetype """ value_mime = value[:-5] if value.endswith("+lazy") else value # ViamImage lazy encodes by default - return cls(value_mime) + try: + return cls(value_mime) + except ValueError: + raise ValueError(f"Invalid mimetype: {value}") @classmethod def from_proto(cls, format: Format.ValueType) -> "CameraMimeType": @@ -70,11 +72,11 @@ class ViamImage: """ _data: bytes - _mime_type: CameraMimeType + _mime_type: str _height: Optional[int] = None _width: Optional[int] = None - def __init__(self, data: bytes, mime_type: CameraMimeType) -> None: + def __init__(self, data: bytes, mime_type: str) -> None: self._data = data self._mime_type = mime_type self._width, self._height = _getDimensions(data, mime_type) @@ -85,7 +87,7 @@ def data(self) -> bytes: return self._data @property - def mime_type(self) -> CameraMimeType: + def mime_type(self) -> str: """The mime type of the image""" return self._mime_type @@ -128,12 +130,12 @@ class NamedImage(ViamImage): """The name of the image """ - def __init__(self, name: str, data: bytes, mime_type: CameraMimeType) -> None: + def __init__(self, name: str, data: bytes, mime_type: str) -> None: self.name = name super().__init__(data, mime_type) -def _getDimensions(image: bytes, mime_type: CameraMimeType) -> Tuple[Optional[int], Optional[int]]: +def _getDimensions(image: bytes, mime_type: str) -> Tuple[Optional[int], Optional[int]]: try: if mime_type == CameraMimeType.JPEG: return _getDimensionsFromJPEG(image) diff --git a/src/viam/services/vision/service.py b/src/viam/services/vision/service.py index 3dd61dc6f..835cf753d 100644 --- a/src/viam/services/vision/service.py +++ b/src/viam/services/vision/service.py @@ -79,8 +79,7 @@ async def GetDetections(self, stream: Stream[GetDetectionsRequest, GetDetections extra = struct_to_dict(request.extra) timeout = stream.deadline.time_remaining() if stream.deadline else None - mime_type = CameraMimeType.from_string(request.mime_type) - image = ViamImage(request.image, mime_type) + image = ViamImage(request.image, request.mime_type) result = await vision.get_detections(image, extra=extra, timeout=timeout) response = GetDetectionsResponse(detections=result) @@ -105,8 +104,7 @@ async def GetClassifications(self, stream: Stream[GetClassificationsRequest, Get extra = struct_to_dict(request.extra) timeout = stream.deadline.time_remaining() if stream.deadline else None - mime_type = CameraMimeType.from_string(request.mime_type) - image = ViamImage(request.image, mime_type) + image = ViamImage(request.image, request.mime_type) result = await vision.get_classifications(image, request.n, extra=extra, timeout=timeout) response = GetClassificationsResponse(classifications=result) diff --git a/tests/test_camera.py b/tests/test_camera.py index 47943e7d9..7ba438311 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -13,7 +13,6 @@ from viam.proto.component.camera import ( CameraServiceStub, DistortionParameters, - Format, GetImageRequest, GetImageResponse, GetImagesRequest, @@ -142,6 +141,7 @@ async def test_get_image(self, camera: MockCamera, service: CameraRPCService, im request = GetImageRequest(name="camera", mime_type=CameraMimeType.PNG) response: GetImageResponse = await client.GetImage(request, timeout=18.1) assert response.image == image.data + assert response.mime_type == CameraMimeType.PNG assert camera.timeout == loose_approx(18.1) # Test empty mime type. Empty mime type should default to response mime type @@ -158,7 +158,7 @@ async def test_get_images(self, camera: MockCamera, service: CameraRPCService, m request = GetImagesRequest(name="camera") response: GetImagesResponse = await client.GetImages(request, timeout=18.1) raw_img = response.images[0] - assert raw_img.format == Format.FORMAT_PNG + assert raw_img.mime_type == CameraMimeType.PNG assert raw_img.source_name == camera.name assert response.response_metadata == metadata assert camera.timeout == loose_approx(18.1)