Recognition of faces on rotated images

[saskra]

I wanted to use this great package to automatically detect when a photo with a face is on its side or upside down so that it can then be rotated correctly. Unfortunately, the faces in such pictures usually don't seem to be recognized at all, as the script below shows. Is there anything that can be done? (Apart from simply trying out the rotations when a face is not recognized). It would probably be ideal to repeat the training and use 90-degree rotations and flips as additional augmentations, but unfortunately I don't have the training database for this.

import os
from io import BytesIO

import cv2
import numpy as np
import requests
from PIL import Image
from facenet_pytorch import MTCNN

# User agent for Wikipedia download
headers = {'User-Agent': 'Mozilla/5.0'}

# Download image from URL
url = 'https://upload.wikimedia.org/wikipedia/commons/a/a0/2007-08-19_Solveig_Hareide_-_Kalv%C3%B8ya.jpg'
response = requests.get(url, headers=headers)
if response.status_code == 200:
	img_data = response.content
	image = Image.open(BytesIO(img_data))
else:
	raise Exception("Image couldn't be downloaded")

# Create a directory to save images
os.makedirs('rotated_images', exist_ok=True)

# Save original image
image.save('rotated_images/original.jpg')

# Convert image to OpenCV format
image_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)

# Define rotations
rotations = {
	'original': image_cv,
	'rotated_90': cv2.rotate(image_cv, cv2.ROTATE_90_CLOCKWISE),
	'rotated_180': cv2.rotate(image_cv, cv2.ROTATE_180),
	'rotated_270': cv2.rotate(image_cv, cv2.ROTATE_90_COUNTERCLOCKWISE)
}

# Save rotated images
for name, img in rotations.items():
	cv2.imwrite(f'rotated_images/{name}.jpg', img)

# Initialize MTCNN
mtcnn = MTCNN(keep_all=True)


def detect_orientation(image_cv2):
	image_rgb = cv2.cvtColor(image_cv2, cv2.COLOR_BGR2RGB)
	boxes, probs, landmarks = mtcnn.detect(image_rgb, landmarks=True)

	if landmarks is not None:
		for landmark in landmarks:
			left_eye = landmark[0]
			right_eye = landmark[1]
			nose = landmark[2]
			left_mouth = landmark[3]
			# right_mouth = landmark[4]

			# Calculate the slope of the eye line
			dx = right_eye[0] - left_eye[0]
			dy = right_eye[1] - left_eye[1]
			angle = np.degrees(np.arctan2(dy, dx))

			# Determine the orientation based on eye line and mouth position
			orientation2 = ""
			if -45 <= angle <= 45:
				if left_mouth[1] > nose[1]:
					orientation2 = "Upright"
				else:
					orientation2 = "Upside down"
			elif 45 < angle <= 135:
				orientation2 = "Rotated 90 degrees"
			elif angle > 135 or angle < -135:
				if left_mouth[1] < nose[1]:
					orientation2 = "Upside down"
				else:
					orientation2 = "Upright"
			elif -135 <= angle < -45:
				orientation2 = "Rotated -90 degrees"
			return orientation2
	return "No face detected"


# Detect and print the orientation for each image
for name, img in rotations.items():
	orientation = detect_orientation(img)
	print(f"Orientation of {name}: {orientation}")

My requirements.txt:

python=3.12.3
facenet-pytorch==2.6.0
opencv-python==4.9.0.80
numpy==1.26.4
requests==2.31.0
pillow==10.2.0