From eee37763a33b26ac6704d237d53ae76ed59674d6 Mon Sep 17 00:00:00 2001 From: hexbabe Date: Mon, 25 Aug 2025 11:04:37 -0400 Subject: [PATCH 1/7] Init --- src/viam/components/camera/camera.py | 2 +- src/viam/components/camera/client.py | 10 ++++++++-- src/viam/components/camera/service.py | 5 ++--- src/viam/media/video.py | 21 +++++++++++---------- tests/test_camera.py | 4 ++-- tests/test_media.py | 4 ++-- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/viam/components/camera/camera.py b/src/viam/components/camera/camera.py index 9faeae9d5..8f114dd53 100644 --- a/src/viam/components/camera/camera.py +++ b/src/viam/components/camera/camera.py @@ -63,7 +63,7 @@ async def get_image( ... @abc.abstractmethod - async def get_images(self, *, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: + async def get_images(self, *, extra: Optional[Dict[str, Any]] = None, filter_source_names: Optional[List[str]] = None, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: """Get simultaneous images from different imagers, along with associated metadata. This should not be used for getting a time series of images from the same imager. diff --git a/src/viam/components/camera/client.py b/src/viam/components/camera/client.py index 45feacf32..5f4c45a74 100644 --- a/src/viam/components/camera/client.py +++ b/src/viam/components/camera/client.py @@ -46,15 +46,21 @@ async def get_image( async def get_images( self, *, + extra: Optional[Dict[str, Any]] = None, + filter_source_names: Optional[List[str]] = None, timeout: Optional[float] = None, **kwargs, ) -> Tuple[List[NamedImage], ResponseMetadata]: md = kwargs.get("metadata", self.Metadata()).proto - request = GetImagesRequest(name=self.name) + request = GetImagesRequest(name=self.name, extra=dict_to_struct(extra), filter_source_names=filter_source_names) response: GetImagesResponse = await self.client.GetImages(request, timeout=timeout, metadata=md) imgs = [] for img_data in response.images: - mime_type = CameraMimeType.from_proto(img_data.format) + if img_data.mime_type: + mime_type = CameraMimeType.from_string(img_data.mime_type) + else: + # TODO: remove this once we deleted the format field + mime_type = CameraMimeType.from_proto(img_data.format) img = NamedImage(img_data.source_name, img_data.image, mime_type) imgs.append(img) resp_metadata: ResponseMetadata = response.response_metadata diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 936271491..1971fef50 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -48,12 +48,11 @@ async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) - camera = self.get_resource(name) timeout = stream.deadline.time_remaining() if stream.deadline else None - images, metadata = await camera.get_images(timeout=timeout, metadata=stream.metadata) + images, metadata = await camera.get_images(timeout=timeout, metadata=stream.metadata, extra=struct_to_dict(request.extra), filter_source_names=request.filter_source_names) img_bytes_lst = [] for img in images: - fmt = img.mime_type.to_proto() img_bytes = img.data - img_bytes_lst.append(Image(source_name=name, format=fmt, image=img_bytes)) + img_bytes_lst.append(Image(source_name=name, mime_type=img.mime_type, image=img_bytes)) response = GetImagesResponse(images=img_bytes_lst, response_metadata=metadata) await stream.send_message(response) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 3b3f7fcc2..0a5037f26 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -1,8 +1,6 @@ from array import array from enum import Enum -from typing import List, Optional, Tuple - -from typing_extensions import Self +from typing import List, Optional, Tuple, Union from viam.errors import NotSupportedError from viam.proto.component.camera import Format @@ -18,7 +16,7 @@ class CameraMimeType(str, Enum): PCD = "pointcloud/pcd" @classmethod - def from_string(cls, value: str) -> Self: + def from_string(cls, value: str) -> Union["CameraMimeType", str]: """Return the mimetype from a string. Args: @@ -28,7 +26,10 @@ def from_string(cls, value: str) -> Self: Self: The mimetype """ value_mime = value[:-5] if value.endswith("+lazy") else value # ViamImage lazy encodes by default - return cls(value_mime) + try: + return cls(value_mime) + except ValueError: + return value_mime @classmethod def from_proto(cls, format: Format.ValueType) -> "CameraMimeType": @@ -70,11 +71,11 @@ class ViamImage: """ _data: bytes - _mime_type: CameraMimeType + _mime_type: str _height: Optional[int] = None _width: Optional[int] = None - def __init__(self, data: bytes, mime_type: CameraMimeType) -> None: + def __init__(self, data: bytes, mime_type: str) -> None: self._data = data self._mime_type = mime_type self._width, self._height = _getDimensions(data, mime_type) @@ -85,7 +86,7 @@ def data(self) -> bytes: return self._data @property - def mime_type(self) -> CameraMimeType: + def mime_type(self) -> str: """The mime type of the image""" return self._mime_type @@ -128,12 +129,12 @@ class NamedImage(ViamImage): """The name of the image """ - def __init__(self, name: str, data: bytes, mime_type: CameraMimeType) -> None: + def __init__(self, name: str, data: bytes, mime_type: str) -> None: self.name = name super().__init__(data, mime_type) -def _getDimensions(image: bytes, mime_type: CameraMimeType) -> Tuple[Optional[int], Optional[int]]: +def _getDimensions(image: bytes, mime_type: str) -> Tuple[Optional[int], Optional[int]]: try: if mime_type == CameraMimeType.JPEG: return _getDimensionsFromJPEG(image) diff --git a/tests/test_camera.py b/tests/test_camera.py index 47943e7d9..7ba438311 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -13,7 +13,6 @@ from viam.proto.component.camera import ( CameraServiceStub, DistortionParameters, - Format, GetImageRequest, GetImageResponse, GetImagesRequest, @@ -142,6 +141,7 @@ async def test_get_image(self, camera: MockCamera, service: CameraRPCService, im request = GetImageRequest(name="camera", mime_type=CameraMimeType.PNG) response: GetImageResponse = await client.GetImage(request, timeout=18.1) assert response.image == image.data + assert response.mime_type == CameraMimeType.PNG assert camera.timeout == loose_approx(18.1) # Test empty mime type. Empty mime type should default to response mime type @@ -158,7 +158,7 @@ async def test_get_images(self, camera: MockCamera, service: CameraRPCService, m request = GetImagesRequest(name="camera") response: GetImagesResponse = await client.GetImages(request, timeout=18.1) raw_img = response.images[0] - assert raw_img.format == Format.FORMAT_PNG + assert raw_img.mime_type == CameraMimeType.PNG assert raw_img.source_name == camera.name assert response.response_metadata == metadata assert camera.timeout == loose_approx(18.1) diff --git a/tests/test_media.py b/tests/test_media.py index 2fb1596b6..1e2c54cbf 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -16,7 +16,7 @@ def test_supported_image(self): b = BytesIO() i.save(b, "PNG") img = ViamImage(b.getvalue(), CameraMimeType.PNG) - assert img._mime_type == CameraMimeType.PNG + assert img._mime_type == "image/png" pil_img = viam_to_pil_image(img) assert pil_img.tobytes() == i.tobytes() @@ -73,7 +73,7 @@ def test_image_conversion(): v_img = pil_to_viam_image(i, CameraMimeType.JPEG) assert isinstance(v_img, ViamImage) - assert v_img.mime_type == CameraMimeType.JPEG + assert v_img.mime_type == "image/jpeg" pil_img = viam_to_pil_image(v_img) v_img2 = pil_to_viam_image(pil_img, CameraMimeType.JPEG) From 48fd8cf98bb0974e2d899974651159cc64f5d832 Mon Sep 17 00:00:00 2001 From: hexbabe Date: Mon, 25 Aug 2025 11:07:26 -0400 Subject: [PATCH 2/7] Assert equality with enum not str --- tests/test_media.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_media.py b/tests/test_media.py index 1e2c54cbf..2fb1596b6 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -16,7 +16,7 @@ def test_supported_image(self): b = BytesIO() i.save(b, "PNG") img = ViamImage(b.getvalue(), CameraMimeType.PNG) - assert img._mime_type == "image/png" + assert img._mime_type == CameraMimeType.PNG pil_img = viam_to_pil_image(img) assert pil_img.tobytes() == i.tobytes() @@ -73,7 +73,7 @@ def test_image_conversion(): v_img = pil_to_viam_image(i, CameraMimeType.JPEG) assert isinstance(v_img, ViamImage) - assert v_img.mime_type == "image/jpeg" + assert v_img.mime_type == CameraMimeType.JPEG pil_img = viam_to_pil_image(v_img) v_img2 = pil_to_viam_image(pil_img, CameraMimeType.JPEG) From 77d3f06df200626c1c35bb754478e93ee397bf67 Mon Sep 17 00:00:00 2001 From: hexbabe Date: Mon, 25 Aug 2025 11:12:18 -0400 Subject: [PATCH 3/7] Cast filter source names to list --- src/viam/components/camera/service.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 1971fef50..39fd8a5b2 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -48,7 +48,12 @@ async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) - camera = self.get_resource(name) timeout = stream.deadline.time_remaining() if stream.deadline else None - images, metadata = await camera.get_images(timeout=timeout, metadata=stream.metadata, extra=struct_to_dict(request.extra), filter_source_names=request.filter_source_names) + images, metadata = await camera.get_images( + timeout=timeout, + metadata=stream.metadata, + extra=struct_to_dict(request.extra), + filter_source_names=list(request.filter_source_names), + ) img_bytes_lst = [] for img in images: img_bytes = img.data From 9d31a24a80eaa6fc6a416ef4e4a9b188cc682b77 Mon Sep 17 00:00:00 2001 From: hexbabe Date: Mon, 25 Aug 2025 11:16:58 -0400 Subject: [PATCH 4/7] Update todo with ticket number --- src/viam/components/camera/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viam/components/camera/client.py b/src/viam/components/camera/client.py index 5f4c45a74..236fc11b4 100644 --- a/src/viam/components/camera/client.py +++ b/src/viam/components/camera/client.py @@ -59,7 +59,7 @@ async def get_images( if img_data.mime_type: mime_type = CameraMimeType.from_string(img_data.mime_type) else: - # TODO: remove this once we deleted the format field + # TODO(RSDK-11728): remove this once we deleted the format field mime_type = CameraMimeType.from_proto(img_data.format) img = NamedImage(img_data.source_name, img_data.image, mime_type) imgs.append(img) From dc6b22f5fb99c634e17029755a4df5bfe6951e90 Mon Sep 17 00:00:00 2001 From: hexbabe Date: Mon, 25 Aug 2025 11:21:27 -0400 Subject: [PATCH 5/7] Change param order --- src/viam/components/camera/camera.py | 2 +- src/viam/components/camera/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/viam/components/camera/camera.py b/src/viam/components/camera/camera.py index 8f114dd53..6174f873d 100644 --- a/src/viam/components/camera/camera.py +++ b/src/viam/components/camera/camera.py @@ -63,7 +63,7 @@ async def get_image( ... @abc.abstractmethod - async def get_images(self, *, extra: Optional[Dict[str, Any]] = None, filter_source_names: Optional[List[str]] = None, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: + async def get_images(self, *, filter_source_names: Optional[List[str]] = None, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: """Get simultaneous images from different imagers, along with associated metadata. This should not be used for getting a time series of images from the same imager. diff --git a/src/viam/components/camera/client.py b/src/viam/components/camera/client.py index 236fc11b4..c0941492a 100644 --- a/src/viam/components/camera/client.py +++ b/src/viam/components/camera/client.py @@ -46,8 +46,8 @@ async def get_image( async def get_images( self, *, - extra: Optional[Dict[str, Any]] = None, filter_source_names: Optional[List[str]] = None, + extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs, ) -> Tuple[List[NamedImage], ResponseMetadata]: From bdf14e407b2e1ec15702622fa7f0f3d09af8d93c Mon Sep 17 00:00:00 2001 From: hexbabe Date: Mon, 25 Aug 2025 15:59:51 -0400 Subject: [PATCH 6/7] Keep format --- src/viam/components/camera/service.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 39fd8a5b2..d78190b12 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -6,6 +6,7 @@ from viam.proto.common import DoCommandRequest, DoCommandResponse, GetGeometriesRequest, GetGeometriesResponse from viam.proto.component.camera import ( CameraServiceBase, + Format, GetImageRequest, GetImageResponse, GetImagesRequest, @@ -19,6 +20,7 @@ ) from viam.resource.rpc_service_base import ResourceRPCServiceBase from viam.utils import dict_to_struct, struct_to_dict +from viam.media.video import CameraMimeType from . import Camera @@ -56,8 +58,14 @@ async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) - ) img_bytes_lst = [] for img in images: + mime_type = CameraMimeType.from_string(img.mime_type) + if isinstance(mime_type, CameraMimeType): + fmt = mime_type.to_proto() + else: + fmt = Format.FORMAT_UNSPECIFIED + img_bytes = img.data - img_bytes_lst.append(Image(source_name=name, mime_type=img.mime_type, image=img_bytes)) + img_bytes_lst.append(Image(source_name=name, mime_type=img.mime_type, format=fmt, image=img_bytes)) response = GetImagesResponse(images=img_bytes_lst, response_metadata=metadata) await stream.send_message(response) From e8cd3372a8d96f7c92a4397bb6fdba8185ea6d50 Mon Sep 17 00:00:00 2001 From: hexbabe Date: Mon, 25 Aug 2025 16:27:48 -0400 Subject: [PATCH 7/7] Change from_string usage --- src/viam/components/camera/client.py | 6 +++--- src/viam/components/camera/service.py | 7 ++++--- src/viam/media/video.py | 7 ++++--- src/viam/services/vision/service.py | 6 ++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/viam/components/camera/client.py b/src/viam/components/camera/client.py index c0941492a..493d219ed 100644 --- a/src/viam/components/camera/client.py +++ b/src/viam/components/camera/client.py @@ -41,7 +41,7 @@ async def get_image( md = kwargs.get("metadata", self.Metadata()).proto request = GetImageRequest(name=self.name, mime_type=mime_type, extra=dict_to_struct(extra)) response: GetImageResponse = await self.client.GetImage(request, timeout=timeout, metadata=md) - return ViamImage(response.image, CameraMimeType.from_string(response.mime_type)) + return ViamImage(response.image, response.mime_type) async def get_images( self, @@ -57,10 +57,10 @@ async def get_images( imgs = [] for img_data in response.images: if img_data.mime_type: - mime_type = CameraMimeType.from_string(img_data.mime_type) + mime_type = img_data.mime_type else: # TODO(RSDK-11728): remove this once we deleted the format field - mime_type = CameraMimeType.from_proto(img_data.format) + mime_type = str(CameraMimeType.from_proto(img_data.format)) img = NamedImage(img_data.source_name, img_data.image, mime_type) imgs.append(img) resp_metadata: ResponseMetadata = response.response_metadata diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index d78190b12..7108bb437 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -58,10 +58,11 @@ async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) - ) img_bytes_lst = [] for img in images: - mime_type = CameraMimeType.from_string(img.mime_type) - if isinstance(mime_type, CameraMimeType): + try: + mime_type = CameraMimeType.from_string(img.mime_type) # this can ValueError if the mime_type is not a CameraMimeType fmt = mime_type.to_proto() - else: + except ValueError: + # TODO(RSDK-11728): remove this once we deleted the format field fmt = Format.FORMAT_UNSPECIFIED img_bytes = img.data diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 0a5037f26..e016e5136 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -1,6 +1,7 @@ from array import array from enum import Enum -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple +from typing_extensions import Self from viam.errors import NotSupportedError from viam.proto.component.camera import Format @@ -16,7 +17,7 @@ class CameraMimeType(str, Enum): PCD = "pointcloud/pcd" @classmethod - def from_string(cls, value: str) -> Union["CameraMimeType", str]: + def from_string(cls, value: str) -> Self: """Return the mimetype from a string. Args: @@ -29,7 +30,7 @@ def from_string(cls, value: str) -> Union["CameraMimeType", str]: try: return cls(value_mime) except ValueError: - return value_mime + raise ValueError(f"Invalid mimetype: {value}") @classmethod def from_proto(cls, format: Format.ValueType) -> "CameraMimeType": diff --git a/src/viam/services/vision/service.py b/src/viam/services/vision/service.py index 3dd61dc6f..835cf753d 100644 --- a/src/viam/services/vision/service.py +++ b/src/viam/services/vision/service.py @@ -79,8 +79,7 @@ async def GetDetections(self, stream: Stream[GetDetectionsRequest, GetDetections extra = struct_to_dict(request.extra) timeout = stream.deadline.time_remaining() if stream.deadline else None - mime_type = CameraMimeType.from_string(request.mime_type) - image = ViamImage(request.image, mime_type) + image = ViamImage(request.image, request.mime_type) result = await vision.get_detections(image, extra=extra, timeout=timeout) response = GetDetectionsResponse(detections=result) @@ -105,8 +104,7 @@ async def GetClassifications(self, stream: Stream[GetClassificationsRequest, Get extra = struct_to_dict(request.extra) timeout = stream.deadline.time_remaining() if stream.deadline else None - mime_type = CameraMimeType.from_string(request.mime_type) - image = ViamImage(request.image, mime_type) + image = ViamImage(request.image, request.mime_type) result = await vision.get_classifications(image, request.n, extra=extra, timeout=timeout) response = GetClassificationsResponse(classifications=result)