Fixed Camera orientation (#956)

on iOS device: window.matchMedia('(orientation: portrait)') can be
unreliable.

Author: dlymonkai

packages/camera-web/src/Camera/Camera.tsx

@@ -85,7 +85,6 @@ export function Camera<T extends object>({
   );
   const {
     ref: videoRef,
-    dimensions: streamDimensions,
     previewDimensions,
     error,
     retry,
@@ -98,7 +97,7 @@ export function Camera<T extends object>({
   });
   const { ref: canvasRef, dimensions: canvasDimensions } = useCameraCanvas({
     resolution,
-    streamDimensions,
+    streamDimensions: previewDimensions,
     allowImageUpscaling,
   });
   const takeScreenshot = useCameraScreenshot({
@@ -147,7 +146,7 @@ export function Camera<T extends object>({
         error,
         retry,
         isLoading,
-        dimensions: streamDimensions,
+        dimensions: previewDimensions,
         previewDimensions,
       }}
       cameraPreview={cameraPreview}

packages/camera-web/src/Camera/hooks/useCameraPreview.ts

@@ -1,10 +1,34 @@
 import { useMonitoring } from '@monkvision/monitoring';
-import { RefObject, useEffect, useMemo, useRef } from 'react';
+import { RefObject, useEffect, useMemo, useRef, useState } from 'react';
 import { PixelDimensions } from '@monkvision/types';
 import { useWindowDimensions } from '@monkvision/common';
 import { CameraConfig, getMediaConstraints } from './utils';
 import { UserMediaResult, useUserMedia } from './useUserMedia';
 
+function getPreviewDimensions(
+  refVideo: RefObject<HTMLVideoElement>,
+  windowDimensions: PixelDimensions,
+) {
+  const height = refVideo.current?.videoHeight;
+  const width = refVideo.current?.videoWidth;
+
+  if (!windowDimensions || !height || !width) {
+    return null;
+  }
+  const windowAspectRatio = windowDimensions.width / windowDimensions.height;
+  const streamAspectRatio = width / height;
+
+  return windowAspectRatio >= streamAspectRatio
+    ? {
+        width: windowDimensions.height * streamAspectRatio,
+        height: windowDimensions.height,
+      }
+    : {
+        width: windowDimensions.width,
+        height: windowDimensions.width / streamAspectRatio,
+      };
+}
+
 /**
  * An object containing properties used to handle the camera preview.
  */
@@ -25,36 +49,37 @@ export interface CameraPreviewHandle extends UserMediaResult {
  */
 export function useCameraPreview(config: CameraConfig): CameraPreviewHandle {
   const ref = useRef<HTMLVideoElement>(null);
+  const [previewDimensions, setPreviewDimensions] = useState<PixelDimensions | null>(null);
   const windowDimensions = useWindowDimensions();
   const { handleError } = useMonitoring();
   const userMediaResult = useUserMedia(getMediaConstraints(config), ref);
 
-  const previewDimensions = useMemo(() => {
-    if (!windowDimensions || !userMediaResult.dimensions) {
-      return null;
-    }
-    const windowAspectRatio = windowDimensions.width / windowDimensions.height;
-    const streamAspectRatio = userMediaResult.dimensions.width / userMediaResult.dimensions.height;
+  useEffect(() => {
+    const currentRef = ref.current;
 
-    return windowAspectRatio >= streamAspectRatio
-      ? {
-          width: windowDimensions.height * streamAspectRatio,
-          height: windowDimensions.height,
-        }
-      : {
-          width: windowDimensions.width,
-          height: windowDimensions.width / streamAspectRatio,
-        };
-  }, [windowDimensions, userMediaResult.dimensions]);
+    if (userMediaResult.stream && currentRef) {
+      currentRef.srcObject = userMediaResult.stream;
 
-  useEffect(() => {
-    if (userMediaResult.stream && ref.current) {
-      ref.current.srcObject = userMediaResult.stream;
-      ref.current.onloadedmetadata = () => {
-        ref.current?.play().catch(handleError);
+      const handleMetadata = () => {
+        currentRef?.play().catch(handleError);
+        setPreviewDimensions(getPreviewDimensions(ref, windowDimensions));
+      };
+
+      const handleResize = () => {
+        setPreviewDimensions(getPreviewDimensions(ref, windowDimensions));
       };
+
+      currentRef.onloadedmetadata = handleMetadata;
+      currentRef.onresize = handleResize;
     }
-  }, [userMediaResult.stream]);
+
+    return () => {
+      if (currentRef) {
+        currentRef.onloadedmetadata = null;
+        currentRef.onresize = null;
+      }
+    };
+  }, [windowDimensions, userMediaResult.stream, handleError]);
 
   return useMemo(
     () => ({

packages/camera-web/src/Camera/hooks/useUserMedia.ts

@@ -1,8 +1,7 @@
 import { useMonitoring } from '@monkvision/monitoring';
 import deepEqual from 'fast-deep-equal';
 import { RefObject, useCallback, useEffect, useRef, useState } from 'react';
-import { PixelDimensions } from '@monkvision/types';
-import { isMobileDevice, useIsMounted, useObjectMemo } from '@monkvision/common';
+import { useIsMounted, useObjectMemo } from '@monkvision/common';
 import { analyzeCameraDevices } from './utils';
 
 /**
@@ -17,10 +16,6 @@ export enum InvalidStreamErrorName {
    * The stream had too many video tracks (more than one).
    */
   TOO_MANY_VIDEO_TRACKS = 'TooManyVideoTracks',
-  /**
-   * The stream's video track had no dimensions.
-   */
-  NO_DIMENSIONS = 'NoDimensions',
 }
 
 class InvalidStreamError extends Error {
@@ -105,11 +100,6 @@ export interface UserMediaResult {
    * The resulting video stream. The stream can be null when not initialized or in case of an error.
    */
   stream: MediaStream | null;
-  /**
-   * The dimensions of the resulting camera stream. Note that these dimensions can differ from the ones given in the
-   * stream constraints if they are not supported or available on the current device.
-   */
-  dimensions: PixelDimensions | null;
   /**
    * The error details. If no error has occurred, this object will be null.
    */
@@ -156,31 +146,6 @@ function getStreamDeviceId(stream: MediaStream): string | null {
   return settings.deviceId ?? null;
 }
 
-function swapDimensions(dimensions: PixelDimensions): PixelDimensions {
-  return {
-    width: dimensions.height,
-    height: dimensions.width,
-  };
-}
-
-function getStreamDimensions(stream: MediaStream, checkOrientation: boolean): PixelDimensions {
-  const { width, height } = getStreamVideoTrackSettings(stream);
-  if (!width || !height) {
-    throw new InvalidStreamError(
-      'Unable to set up the Monk camera screenshoter because the video stream does not have the properties width and height defined.',
-      InvalidStreamErrorName.NO_DIMENSIONS,
-    );
-  }
-  const dimensions = { width, height };
-  if (!isMobileDevice() || !checkOrientation) {
-    return dimensions;
-  }
-
-  const isStreamInPortrait = width < height;
-  const isDeviceInPortrait = window.matchMedia('(orientation: portrait)').matches;
-  return isStreamInPortrait !== isDeviceInPortrait ? swapDimensions(dimensions) : dimensions;
-}
-
 /**
  * React hook that wraps the `navigator.mediaDevices.getUserMedia` browser function in order to add React logic layers
  * and utility tools :
@@ -205,7 +170,6 @@ export function useUserMedia(
 ): UserMediaResult {
   const streamRef = useRef<MediaStream | null>(null);
   const [stream, setStream] = useState<MediaStream | null>(null);
-  const [dimensions, setDimensions] = useState<PixelDimensions | null>(null);
   const [isLoading, setIsLoading] = useState(false);
   const [error, setError] = useState<UserMediaError | null>(null);
   const [availableCameraDevices, setAvailableCameraDevices] = useState<MediaDeviceInfo[]>([]);
@@ -278,10 +242,9 @@ export function useUserMedia(
     if (isMounted()) {
       setStream(str);
       streamRef.current = str;
-      setDimensions(getStreamDimensions(str, true));
-      setIsLoading(false);
-      setAvailableCameraDevices(deviceDetails.availableDevices);
       setSelectedCameraDeviceId(getStreamDeviceId(str));
+      setAvailableCameraDevices(deviceDetails.availableDevices);
+      setIsLoading(false);
     }
     return str;
   }, [stream, constraints]);
@@ -323,17 +286,6 @@ export function useUserMedia(
     }
   }, [constraints, stream, error, isLoading, lastConstraintsApplied, getUserMedia, videoRef]);
 
-  useEffect(() => {
-    if (stream && videoRef && videoRef.current) {
-      // eslint-disable-next-line no-param-reassign
-      videoRef.current.onresize = () => {
-        if (isMounted()) {
-          setDimensions(getStreamDimensions(stream, false));
-        }
-      };
-    }
-  }, [stream, videoRef]);
-
   useEffect(() => {
     return () => {
       streamRef.current?.getTracks().forEach((track) => {
@@ -345,7 +297,6 @@ export function useUserMedia(
   return useObjectMemo({
     getUserMedia,
     stream,
-    dimensions,
     error,
     retry,
     isLoading,

packages/camera-web/test/Camera/Camera.test.tsx

@@ -88,12 +88,12 @@ describe('Camera component', () => {
       <Camera allowImageUpscaling={allowImageUpscaling} resolution={resolution} />,
     );
 
-    const streamDimensions = (useCameraPreview as jest.Mock).mock.results[0].value.dimensions;
+    const { previewDimensions } = (useCameraPreview as jest.Mock).mock.results[0].value;
 
     expect(useCameraCanvas).toHaveBeenCalledWith({
       allowImageUpscaling,
       resolution,
-      streamDimensions,
+      previewDimensions,
     });
     unmount();
   });
@@ -224,7 +224,7 @@ describe('Camera component', () => {
         error: useCameraPreviewResultMock.error,
         retry: useCameraPreviewResultMock.retry,
         isLoading: useCameraPreviewResultMock.isLoading || useTakePictureResultMock.isLoading,
-        dimensions: useCameraPreviewResultMock.dimensions,
+        dimensions: useCameraPreviewResultMock.previewDimensions,
       },
       cameraPreview: expect.anything(),
     });