diff --git a/.gitignore b/.gitignore index f9ebcf60..18e0d539 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ dist /tests/artifacts /tests/htmlcov/ /tests/report* + +# Downloaded datasets +/notebooks/use_cases/data/ood_detection \ No newline at end of file diff --git a/notebooks/use_cases/103_out_of_distribution_detection.ipynb b/notebooks/use_cases/103_out_of_distribution_detection.ipynb new file mode 100644 index 00000000..8068127e --- /dev/null +++ b/notebooks/use_cases/103_out_of_distribution_detection.ipynb @@ -0,0 +1,759 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OOD Detection using the Intel® Geti™ SDK\n", + "\n", + "This notebook shows the out-of-distribution (OOD) detection using the [kNN-based OOD detection](https://arxiv.org/abs/2204.06507) method for an image classification task. In this example, a classifier trained on the CUB-200-2011 dataset is used." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1: Preparing the dataset for training the classifier" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1 Downloading and extracting the CUB-200-2011 dataset\n", + "\n", + "The [CUB-200-2011](https://www.vision.caltech.edu/datasets/cub_200_2011/) dataset is a dataset of 200 classes of birds. In this notebook, we use 90% of the dataset for training the classifier and the rest 10% as the test set for in-distribution. The same images with corruption (e.g. motion blurred) are used as the out-of-distribution dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import os\n", + "import tarfile\n", + "from urllib import request\n", + "\n", + "import splitfolders\n", + "from tqdm import tqdm\n", + "\n", + "# Provide the dataset (extracted/to be extracted) path here. If the dataset is not downloaded, it will be downloaded and extracted.\n", + "data_dir = \"./use_cases/data/ood_detection/cub200\"\n", + "\n", + "# CUB-200-2011 example\n", + "cub200_tar = os.path.join(data_dir, \"CUB_200_2011.tgz\")\n", + "# If the dataset is not downloaded, download it\n", + "if not os.path.exists(cub200_tar):\n", + " cub200_url = \"https://data.caltech.edu/records/65de6-vp158/files/CUB_200_2011.tgz\"\n", + " os.makedirs(data_dir, exist_ok=True)\n", + " print(f\"Downloading dataset to {cub200_tar}\")\n", + " request.urlretrieve(cub200_url, cub200_tar)\n", + " print(\"Download complete.\")\n", + "\n", + "# Extract the dataset unless it is already extracted.\n", + "if not os.path.exists(os.path.join(data_dir, \"CUB_200_2011\")):\n", + " print(f\"Extracting dataset to f{os.path.join(data_dir,'CUB_200_2011')}\")\n", + " with tarfile.open(cub200_tar, \"r:gz\") as tar:\n", + " for member in tqdm(iterable=tar.getmembers(), total=len(tar.getmembers())):\n", + " tar.extract(member=member, path=data_dir)\n", + "\n", + "# Split the dataset for training and test purposes - Split used is 80:20 and can be changed.\n", + "# The trainset will be further split into train,val and test automatically on the Geti instance.\n", + "dataset_dir = os.path.join(data_dir, \"CUB_200_2011_split\")\n", + "if not os.path.exists(dataset_dir):\n", + " print(f\"Splitting dataset into train and test at {dataset_dir}\")\n", + " splitfolders.ratio(\n", + " os.path.join(data_dir, \"CUB_200_2011\", \"images\"),\n", + " output=dataset_dir,\n", + " seed=117,\n", + " ratio=(0.9, 0.1),\n", + " group_prefix=None,\n", + " move=False,\n", + " )\n", + " os.rename(os.path.join(dataset_dir, \"val\"), os.path.join(dataset_dir, \"id_test\"))\n", + "\n", + "print(\"Dataset ready to be used\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2: Train a classifier on the Intel® Geti™ instance." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1: Creating a Geti object and authenticating it.\n", + "For authentication, you need to have a .env file configuration file placed in the same directory of this notebook. More details [here](https://github.com/openvinotoolkit/geti-sdk/tree/main/notebooks#authentication)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from geti_sdk.utils import get_server_details_from_env\n", + "\n", + "geti_server_configuration = get_server_details_from_env()\n", + "\n", + "from geti_sdk import Geti\n", + "\n", + "geti = Geti(server_config=geti_server_configuration)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2 : Creating a project and uploading the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from geti_sdk.annotation_readers import DirectoryTreeAnnotationReader\n", + "\n", + "PROJECT_NAME = \"CUB200-910\" # Name of the project on the Geti instance" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Skip the next step if the project is already created on the Geti instance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "annotation_reader = DirectoryTreeAnnotationReader(\n", + " base_data_folder=os.path.join(dataset_dir, \"train\")\n", + ")\n", + "print(\n", + " f\"# of images for training the classifier : {len(annotation_reader.get_data_filenames())}\"\n", + ")\n", + "print(f\"# of classes : {len(annotation_reader.get_all_label_names())}\")\n", + "\n", + "\n", + "project = geti.create_single_task_project_from_dataset(\n", + " project_name=PROJECT_NAME,\n", + " project_type=\"classification\",\n", + " path_to_images=os.path.join(dataset_dir, \"train\"),\n", + " annotation_reader=annotation_reader,\n", + " enable_auto_train=False,\n", + ")\n", + "print(project.summary)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-21T12:37:04.931363700Z", + "start_time": "2023-06-21T12:36:58.216222100Z" + } + }, + "source": [ + "### 2.3 Train the classifier\n", + "We choose the EfficientNet-V2-S algorithm for training the classifier. The list of available algorithms can be obtained using the following code snippet." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, + "pycharm": { + "is_executing": true + } + }, + "outputs": [], + "source": [ + "from geti_sdk.rest_clients import ProjectClient, TrainingClient\n", + "\n", + "project_client = ProjectClient(session=geti.session, workspace_id=geti.workspace_id)\n", + "project = project_client.get_project_by_name(project_name=PROJECT_NAME)\n", + "training_client = TrainingClient(\n", + " session=geti.session, workspace_id=geti.workspace_id, project=project\n", + ")\n", + "\n", + "task = project.get_trainable_tasks()[0]\n", + "available_algorithms = training_client.get_algorithms_for_task(task=task)\n", + "print(available_algorithms.summary)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-20T10:38:16.766807100Z", + "start_time": "2023-06-20T10:09:25.210683Z" + } + }, + "source": [ + "Skip this step if the classifier is already trained." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "algorithm = available_algorithms.get_by_name(name=\"EfficientNet-V2-S\")\n", + "status = training_client.get_status()\n", + "print(status.summary)\n", + "\n", + "job = training_client.train_task(\n", + " algorithm=algorithm,\n", + " task=task,\n", + ")\n", + "training_client.monitor_jobs([job])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-21T12:37:19.534902400Z", + "start_time": "2023-06-21T12:37:18.918252700Z" + } + }, + "source": [ + "### 2.4 Downloading the trained classifier model for inference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from geti_sdk.demos.demo_projects.utils import ensure_project_is_trained\n", + "from geti_sdk.rest_clients.model_client import ModelClient\n", + "\n", + "# Confirm is the model is trained.\n", + "_ = ensure_project_is_trained(geti=geti, project=project)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "model_client = ModelClient(\n", + " session=geti.session, workspace_id=geti.workspace_id, project=project\n", + ")\n", + "models = model_client.get_all_active_models()\n", + "\n", + "# We need the model which has xai enabled - this allows us to get the feature vector from the model.\n", + "model_index = next(\n", + " (\n", + " index\n", + " for index, model in enumerate(models[0].optimized_models)\n", + " if model.has_xai_head\n", + " ),\n", + " None,\n", + ")\n", + "if model_index is None:\n", + " raise Exception(\n", + " \"No model with XAI head found! Please check if the project has such a model on the Geti instance\"\n", + " )\n", + "\n", + "model_for_deployment = models[0].optimized_models[model_index]\n", + "model_accuracy = model_for_deployment.performance.score\n", + "\n", + "print(\n", + " f\"Model for deployment : {model_for_deployment.name} (accuracy : {model_accuracy*100:.2f} %)\"\n", + ")\n", + "deployment = geti.deploy_project(\n", + " project_name=PROJECT_NAME, models=[model_for_deployment]\n", + ")\n", + "deployment.load_inference_models(device=\"CPU\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-22T09:05:44.074454200Z", + "start_time": "2023-06-22T08:57:27.920358700Z" + } + }, + "source": [ + "## 3 : Out-of-distribution dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-22T09:05:53.950164300Z", + "start_time": "2023-06-22T09:05:49.816967600Z" + } + }, + "source": [ + "### 3.1: Generating the out-of-distribution dataset\n", + "We create the out-of-distribution by applying corruptions (e.g. motion blur) on the test set of in-distribution images. The strength of the corruptions is tuned until the test set has a classification accuracy of x%, i.e., half of the test set is classified incorrectly.\n", + "\n", + "The possible corruptions are: `gaussian_blur`, `motion_blur`, `fake_snow`, `cut_out` and `poisson_noise`.\n", + "\n", + "You can set the `generate_ood_images` flag to `False` and set the `ood_images_path` to the path of the out-of-distribution images if you want to use a different set of images as OOD." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "id_images_path: str = os.path.join(dataset_dir, \"id_test\")\n", + "ood_images_path: str = os.path.join(\n", + " dataset_dir, \"ood_test\"\n", + ") # Set this to the path of the OOD images if you want to use a different set of images as OOD." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "generate_ood_images: bool = True\n", + "\n", + "if generate_ood_images:\n", + " from notebooks.use_cases.utils import TransformImages\n", + "\n", + " transform_images = TransformImages(corruption_type=\"motion_blur\")\n", + " ood_images_path = transform_images.generate_ood_dataset_by_corruption(\n", + " geti_deployment=deployment,\n", + " source_path=id_images_path,\n", + " dest_path=ood_images_path,\n", + " desired_accuracy=50,\n", + " desired_accuracy_tol=3.0,\n", + " show_progress=True,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from notebooks.use_cases.utils import display_sample_images_in_folder\n", + "\n", + "display_sample_images_in_folder(\n", + " id_images_path, n_images=10, title=\"In-distribution images\"\n", + ")\n", + "display_sample_images_in_folder(\n", + " ood_images_path, n_images=10, title=\"Out-of-distribution images\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-20T12:48:00.935602100Z", + "start_time": "2023-06-20T12:47:56.304613900Z" + } + }, + "source": [ + "## 4: OOD Detection\n", + "\n", + "We are using a simple [kNN-based OOD detection method](https://arxiv.org/abs/2204.06507). The OOD score is calculated as the distance to the kth nearest neighbour in the feature space (of known in-distribution images)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-22T08:24:21.893741100Z", + "start_time": "2023-06-22T08:24:19.340621400Z" + } + }, + "source": [ + "### 4.1 : Calibration - Calculating the OOD score threshold" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from utils import extract_features_from_imageclient\n", + "\n", + "from geti_sdk.rest_clients import ImageClient\n", + "\n", + "image_client = ImageClient(\n", + " session=geti.session, workspace_id=geti.workspace_id, project=project\n", + ")\n", + "\n", + "# set the number of images to be used for calculating the OOD score threshold. The images used for training the classifier are used for calibration.\n", + "# higher n_images_for_calib --> more accurate OOD score threshold, but also more time to get features of images\n", + "n_images_for_calib = -1 # set to -1 to use all images\n", + "\n", + "features_id = extract_features_from_imageclient(\n", + " deployment=deployment,\n", + " image_client=image_client,\n", + " geti_session=geti.session,\n", + " n_images=n_images_for_calib,\n", + " normalise_feats=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "import faiss\n", + "\n", + "dknn_k = 6 # number of nearest neighbours to be used for calculating the OOD score threshold. This is a hyperparameter for the DkNN OOD detection method. A number in the range of 4-10 has given good results in our experiments. Should be lower than the number of classes in the dataset.\n", + "\n", + "index_flat = faiss.IndexFlatL2(\n", + " features_id.shape[1]\n", + ") # Indexing the features of the ID images\n", + "index_flat.add(features_id.astype(np.float32))\n", + "dists, nns = index_flat.search(\n", + " features_id.astype(np.float32), dknn_k + 1\n", + ") # Calculating the distances to the k nearest neighbours among the set of ID images\n", + "\n", + "# Calculating the OOD score threshold\n", + "n_percentile = 99.9\n", + "\n", + "# We set the distance threshold such that at least n_percentile% of known ID images are classified correctly.\n", + "# Higher number --> more strict OOD score threshold, more true positives, but also more false positives\n", + "ood_score_threshold = np.percentile(dists[:, dknn_k].flatten(), n_percentile)\n", + "print(f\"OOD Threshold distance : {ood_score_threshold:.2f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-22T09:07:51.619209300Z", + "start_time": "2023-06-22T09:07:51.202295100Z" + } + }, + "source": [ + "### 4.2 : OOD Detection - Calculating the OOD scores for the ID and OOD test images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from utils import extract_features_from_img_folder\n", + "\n", + "# test images for ID and OOD\n", + "id_images_path = os.path.join(dataset_dir, \"id_test\")\n", + "\n", + "id_features = extract_features_from_img_folder(\n", + " deployment=deployment, images_folder_path=id_images_path, normalise_feats=True\n", + ")\n", + "\n", + "ood_features = extract_features_from_img_folder(\n", + " deployment=deployment, images_folder_path=ood_images_path, normalise_feats=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "# Performing the knn search with k = dknn_k\n", + "scores_id, _ = index_flat.search(id_features.astype(np.float32), k=dknn_k)\n", + "scores_ood, _ = index_flat.search(ood_features.astype(np.float32), k=dknn_k)\n", + "\n", + "# Take the highest distance --> this is the distance to the kth neighbour.\n", + "# Taking a negative of the scores as we would want the ID images to have higher value while plotting the results\n", + "scores_id = -scores_id[:, -1]\n", + "scores_ood = -scores_ood[:, -1]\n", + "\n", + "scores_concat = np.concatenate((scores_id, scores_ood))\n", + "ground_truth_id = np.ones(scores_id.shape[0])\n", + "ground_truth_ood = np.zeros(scores_ood.shape[0])\n", + "ground_truth = np.concatenate((ground_truth_id, ground_truth_ood))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-22T09:07:54.418411700Z", + "start_time": "2023-06-22T09:07:53.955948500Z" + } + }, + "source": [ + "## 5 : OOD Detection - Plotting the results" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-06-22T09:08:25.177320400Z", + "start_time": "2023-06-22T09:08:17.939839400Z" + } + }, + "source": [ + "### 5.1 : Results - ROC curve" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from matplotlib import pyplot as plt\n", + "from sklearn import metrics\n", + "\n", + "fpr, tpr, thresholds_roc = metrics.roc_curve(ground_truth, scores_concat)\n", + "precision, recall, thresholds_pr = metrics.precision_recall_curve(\n", + " ground_truth, scores_concat\n", + ")\n", + "auroc = metrics.auc(fpr, tpr)\n", + "fig = plt.figure(figsize=(5, 5))\n", + "plt.plot(fpr, tpr, color=\"#003e6d\")\n", + "plt.text(0.8, 0.3, f\"AUROC = {auroc:.4f}\", fontsize=12, ha=\"center\")\n", + "plt.title(\"ROC \\n(ID images as positive examples)\")\n", + "plt.xlabel(\"False Positive Rate\")\n", + "plt.ylabel(\"True Positive Rate\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.2 : Results - Confusion Matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "import seaborn as sns\n", + "\n", + "predictions = scores_concat > (-ood_score_threshold)\n", + "confusion_matrix = metrics.confusion_matrix(ground_truth, predictions)\n", + "ax = sns.heatmap(\n", + " confusion_matrix,\n", + " annot=True,\n", + " fmt=\"d\",\n", + ")\n", + "ax.set_xlabel(\"Predicted\", fontsize=14, labelpad=20)\n", + "ax.xaxis.set_ticklabels([\"(OOD)\", \"(ID)\"])\n", + "ax.set_ylabel(\"Ground Truth\", fontsize=14, labelpad=20)\n", + "ax.yaxis.set_ticklabels([\"(OOD)\", \"(ID)\"])\n", + "ax.set_title(\"Confusion Matrix - ID v OOD\", fontsize=14, pad=20)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.3 Results - Displaying mis-classified examples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from utils import show_top_n_misclassifications\n", + "\n", + "# The following plot shows the overlap in scores between in- and out-of-distribution images.\n", + "\n", + "fig = plt.figure(figsize=(5, 5))\n", + "sns.kdeplot(scores_id, fill=True, color=\"#0068b5\", label=\"ID\")\n", + "sns.kdeplot(scores_ood, fill=True, color=\"#e96115\", label=\"OOD\")\n", + "plt.axvline(x=-ood_score_threshold, color=\"#001220\", linestyle=\"--\")\n", + "plt.xlabel(\"OOD Score\")\n", + "plt.ylabel(\"Density\")\n", + "plt.title(\"OOD Score Distribution\")\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "# The following figures show the top n misclassified ID and OOD images.\n", + "show_top_n_misclassifications(\n", + " images_dir=id_images_path,\n", + " scores=scores_id,\n", + " type_of_samples=\"id\",\n", + " n_images=9,\n", + ")\n", + "show_top_n_misclassifications(\n", + " images_dir=ood_images_path,\n", + " scores=scores_ood,\n", + " type_of_samples=\"ood\",\n", + " n_images=9,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "# delete the project on the Geti instance if required (this can not be undone)\n", + "# project_client.delete_project(project=project_name, requires_confirmation=False)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/use_cases/utils/__init__.py b/notebooks/use_cases/utils/__init__.py index 8d33e6a0..4d89ff9b 100644 --- a/notebooks/use_cases/utils/__init__.py +++ b/notebooks/use_cases/utils/__init__.py @@ -15,11 +15,26 @@ # and limitations under the License. -from .image import display_image_in_notebook, simulate_low_light_image +from .image import ( + TransformImages, + display_image_in_notebook, + display_sample_images_in_folder, + extract_features_from_imageclient, + extract_features_from_img_folder, + get_image_paths, + simulate_low_light_image, +) +from .ood_detect import show_top_n_misclassifications from .upload import Uploader from .video import VideoPlayer __all__ = [ + "get_image_paths", + "show_top_n_misclassifications", + "extract_features_from_imageclient", + "display_sample_images_in_folder", + "extract_features_from_img_folder", + "TransformImages", "simulate_low_light_image", "display_image_in_notebook", "VideoPlayer", diff --git a/notebooks/use_cases/utils/image.py b/notebooks/use_cases/utils/image.py index d6db251b..0a1ebc0e 100644 --- a/notebooks/use_cases/utils/image.py +++ b/notebooks/use_cases/utils/image.py @@ -11,14 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions # and limitations under the License. +import os +from math import sqrt from typing import Union import cv2 +import imgaug.augmenters as iaa +import matplotlib.pyplot as plt import numpy as np from IPython.display import display from PIL import Image as PILImage +from tqdm import tqdm from geti_sdk.data_models import Image +from geti_sdk.deployment import Deployment +from geti_sdk.http_session import GetiSession +from geti_sdk.rest_clients import ImageClient def simulate_low_light_image( @@ -61,16 +69,489 @@ def display_image_in_notebook(image: Union[np.ndarray, Image], bgr: bool = True) :param bgr: True if the image has its channels in BGR order, False if it is in RGB order. Defaults to True """ + new_image = _image_to_np(image) + if bgr: + result = cv2.cvtColor(new_image, cv2.COLOR_BGR2RGB) + else: + result = new_image + img = PILImage.fromarray(result) + display(img) + + +def _image_to_np( + image: Union[np.ndarray, Image], +) -> np.ndarray: + """ + Make sure an image is a numpy array. + """ if isinstance(image, np.ndarray): new_image = image.copy() elif isinstance(image, Image): new_image = image.numpy.copy() else: raise TypeError(f"Unsupported image type '{type(image)}'") + return new_image - if bgr: - result = cv2.cvtColor(new_image, cv2.COLOR_BGR2RGB) + +class TransformImages: + """ + Class for applying corruptions/augmentations to images and datasets + """ + + def __init__( + self, + corruption_type: str = "motion_blur", + ): + """ + :param corruption_type: The type of corruption to apply. + + """ + self.corruption_type = corruption_type + self._SEED = 117 + self._SUPPORTED_CORRUPTIONS = [ + "motion_blur", + "gaussian_blur", + "cut_out", + "fake_snow", + "poisson_noise", + ] + if self.corruption_type not in self._SUPPORTED_CORRUPTIONS: + raise ValueError( + f"Unsupported corruption type '{self.corruption_type}, " + f"supported corruption types are {self._SUPPORTED_CORRUPTIONS}" + ) + + self._corruption_strength_range = (1, 300) + if self.corruption_type in ["cut_out", "fake_snow"]: + self._corruption_strength_range = (0.001, 1) + elif self.corruption_type == "poisson_noise": + self._corruption_strength_range = ( + 1, + 110, + ) # Increase this if the desired accuracy is not reached + elif self.corruption_type == "gaussian_blur": + self._corruption_strength_range = ( + 1, + 8, + ) # Increase this if the desired accuracy is not reached + elif self.corruption_type == "motion_blur": + self._corruption_strength_range = ( + 1, + 110, + ) # Increase this if the desired accuracy is not reached + + def _compose_imgaug_corruption(self, corruption_strength: Union[float, int]): + transform = None + if self.corruption_type == "motion_blur": + transform = iaa.MotionBlur( + k=int(round(corruption_strength)), + angle=45, + direction=0.5, + seed=self._SEED, + ) + + elif self.corruption_type == "gaussian_blur": + transform = iaa.GaussianBlur(sigma=corruption_strength, seed=self._SEED) + + elif self.corruption_type == "cut_out": + transform = iaa.Cutout( + nb_iterations=2, # No. of boxes per image + size=corruption_strength, + squared=False, + seed=self._SEED, + position="uniform", # "uniform" - random, "normal" - center of the image + fill_mode="constant", + cval=0, + ) # fill with black boxes + + elif self.corruption_type == "fake_snow": + # SnowFlakes from ImgAug can give error depending on the numpy version you have. + transform = iaa.Snowflakes( + density=0.075, + density_uniformity=1.0, + flake_size=corruption_strength, + flake_size_uniformity=0.5, + angle=45, + speed=0.025, + seed=self._SEED, + ) + + elif self.corruption_type == "poisson_noise": + transform = iaa.AdditivePoissonNoise( + lam=corruption_strength, per_channel=True, seed=self._SEED + ) + + return transform + + def _apply_corruption_on_image( + self, + image: Union[np.ndarray, Image], + corruption_strength: Union[float, int] = 0.5, + ) -> np.ndarray: + """ + Apply a corruption to an image. + + :param image: Original image + :param corruption_strength: The strength of the corruption. Ignored if corruption_type + is an albumentations.Compose object. + :return: Copy of the image with augmentation/corruption applied. + """ + input_image = _image_to_np(image) + transformation = self._compose_imgaug_corruption( + corruption_strength=corruption_strength + ) + transformed_image = transformation(image=input_image) + return transformed_image + + def apply_corruption_on_folder( + self, + source_path: str, + dest_path: str, + corruption_strength: Union[float, int], + show_progress: bool = True, + ) -> str: + """ + Apply a corruption to all images in a folder. + + :param source_path: Path to the folder containing the images (dataset format). + :param dest_path: Path to the folder where the transformed images will be saved. + :param corruption_strength: The strength of the corruption to apply. Range depends on the corruption type. + :param show_progress: Whether to show a progress bar or not. + """ + if not os.path.exists(source_path): + raise FileNotFoundError(f"Source path '{source_path}' does not exist") + if not os.path.exists(dest_path): + os.makedirs(dest_path) + for folder_name in tqdm( + os.listdir(source_path), + disable=not show_progress, + desc="Applying Corruption", + ): + class_folder_path = os.path.join(source_path, folder_name) + # loop through images in each class folder + for image_name in os.listdir(class_folder_path): + image = cv2.imread(os.path.join(class_folder_path, image_name)) + dest_folder = os.path.join(dest_path, folder_name) + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + + transformed_image = self._apply_corruption_on_image( + image, corruption_strength=corruption_strength + ) + # Save the transformed image to the destination folder + cv2.imwrite(os.path.join(dest_folder, image_name), transformed_image) + return dest_path + + def generate_ood_dataset_by_corruption( + self, + geti_deployment: Deployment, + source_path: str, + dest_path: str = None, + desired_accuracy: float = 50, + desired_accuracy_tol=3.0, + show_progress: bool = True, + ) -> str: + """ + Generate a dataset of corrupted images from a source dataset of clean images. + The corruption is applied until the classification accuracy (tp/tp+fp) on the + generated dataset reached the desired accuracy. + + :geti_deployment: The trained geti deployment (model) to use for classification. + :source_path: The path to the source dataset of clean images. The dataset is + required to have the following structure: + + source_path/ + class_1/ + image_1.jpg + image_2.jpg + ... + class_2/ + image_1.jpg + image_2.jpg + ... + ... + + :corruption_to_apply: The type of corruption to apply. Currently supported corruptions + are: self._SUPPORTED_CORRUPTIONS + :desired_accuracy: The desired classification accuracy in percentage on the generated dataset. + A 50% accuracy means that the model is not able to correctly classify half of the + images in the dataset. + :show_progress: If True, a progress bar will be displayed. + :return: The path to the generated dataset. + """ + print( + f"Generating OOD dataset by applying {self.corruption_type} corruption on {source_path}" + ) + dataset_folder_name = os.path.basename(source_path) + + if dest_path is None: + dest_path = os.path.join( + os.path.dirname(source_path), + f"{dataset_folder_name}_{self.corruption_type}_{desired_accuracy:.0f}", + ) + + accuracy = calc_classification_accuracy( + dataset_path=source_path, + deployment=geti_deployment, + show_progress=show_progress, + ) + if accuracy < desired_accuracy: + print(f"Maximum possible accuracy : {accuracy:.2f} %") + print(f"Can not reach desired accuracy of {desired_accuracy:.2f} %") + return source_path + if show_progress: + print(f"Accuracy without any corruptions applied : {accuracy:.2f} %") + corruption_strength = self._corruption_strength_range[0] + while abs(accuracy - desired_accuracy) > desired_accuracy_tol: + corruption_strength = self._update_corruption_strength( + desired_accuracy=desired_accuracy, + current_accuracy=accuracy, + current_strength=corruption_strength, + ) + self.apply_corruption_on_folder( + source_path=source_path, + dest_path=dest_path, + corruption_strength=corruption_strength, + show_progress=show_progress, + ) + accuracy = calc_classification_accuracy( + dataset_path=dest_path, + deployment=geti_deployment, + show_progress=show_progress, + ) + if show_progress: + print(f"Current accuracy: {accuracy:.2f}") + + if abs(accuracy - desired_accuracy) < desired_accuracy_tol: + print(f"Corrupted dataset generated with accuracy {accuracy:.2f} %") + return dest_path + + def _update_corruption_strength( + self, + desired_accuracy: float, + current_accuracy: float, + current_strength: float, + ) -> float: + """ + Update the corruption strength based on the current accuracy and the desired accuracy. + """ + # The weight is used to control the speed of the corruption strength update. + # If the current accuracy is close to the desired accuracy, a smaller weight is used to prevent overshoot. + weight = 1.0 if abs(int((current_accuracy - desired_accuracy))) > 10 else 0.7 + + limit = self._corruption_strength_range + + accuracy_diff = ((current_accuracy - desired_accuracy) / 100) * ( + limit[1] - limit[0] + ) + updated_parameter = current_strength + accuracy_diff * weight + + updated_parameter = max(limit[0], min(limit[1], updated_parameter)) + return updated_parameter + + +def calc_classification_accuracy( + dataset_path: str, + deployment: Deployment, + show_progress: bool = True, +) -> float: + """ + Calculate the classification accuracy of a dataset using a trained Geti deployment. + :param dataset_path: The path to the (test) dataset. The dataset is supposed to have the following structure: + dataset_path/ + class_1/ + image_1.jpg + image_2.jpg + ... + class_2/ + image_1.jpg + image_2.jpg + ... + ... + :param deployment: The trained Geti deployment (model) to use for classification. + :param show_progress: If True, a progress bar will be displayed. + :return: The classification accuracy of the dataset in percentage. + """ + id_images_dict = get_image_paths(src_dir=dataset_path, images_dict=None, label=None) + correct_classifications = 0 + for image_path, label in tqdm( + id_images_dict.items(), disable=not show_progress, desc="Calculating Accuracy" + ): + numpy_image = cv2.imread(image_path) + numpy_rgb = cv2.cvtColor(numpy_image, cv2.COLOR_BGR2RGB) + prediction = deployment.infer(numpy_rgb) + pred_label = prediction.annotations[0].labels[0].name + if pred_label == label: + correct_classifications += 1 + accuracy = correct_classifications / len(id_images_dict) + return accuracy * 100 + + +def get_image_paths(src_dir, images_dict=None, label=None): + """ + Recursively retrieve the paths of all images in a directory. + + :param src_dir: The path to the directory containing the images. + :param images_dict: A dictionary containing the image paths and their corresponding labels. + If not None, the retrieved image paths will be added to this dictionary. + :param label: The label to assign to the images in the directory. + If None, the directory name will be used as label. + """ + if images_dict is None: + images_dict = {} + if label is None: + label = os.path.basename(src_dir) + for img_file in os.listdir(src_dir): + img_path = os.path.join(src_dir, img_file) + if os.path.isdir(img_path): + get_image_paths(img_path, images_dict, label=None) + else: + images_dict[img_path] = label + return images_dict + + +def extract_features_from_imageclient( + deployment: Deployment, + image_client: ImageClient, + geti_session: GetiSession, + n_images: int = -1, + normalise_feats: bool = True, +): + """ + Extract feature embeddings from a Geti deployment model for a given number of + images in a geti image_client + + :param deployment: The trained Geti deployment (model) to use for feature extraction. + :param image_client: The Geti ImageClient object containing the images to extract features from. + :param geti_session: The GetiSession object. + :param n_images: The number of images to extract features from. + If -1, all images in the image_client will be used. + :param normalise_feats: If True, the feature embeddings are normalised by dividing each feature + embedding vector by its respective 2nd-order vector norm (vector Euclidean norm) + :return: A numpy array containing the extracted feature embeddings of shape (n_images, feature_len) + """ + print("Retrieving the list of images from the project ...") + images_in_client = image_client.get_all_images() + total_n_images = len(images_in_client) # total number of images in the project + if n_images == -1: + n_images = total_n_images + + n_images = min(n_images, total_n_images) + sample_image = images_in_client[0].get_data(session=geti_session) + prediction = deployment.explain(sample_image) + feature_len = prediction.feature_vector.shape[0] + # pick random images - Stratified sampling not possible yet as we don't have labels in image_client + random_indices = np.random.choice(total_n_images, n_images, replace=False) + features = np.zeros((n_images, feature_len)) + for i, k in tqdm( + enumerate(random_indices), total=n_images, desc="Extracting features" + ): + image_numpy = images_in_client[k].get_data(session=geti_session) + prediction = deployment.explain(image_numpy) + features[i] = prediction.feature_vector + + if normalise_feats: + features = features / np.linalg.norm(features, axis=1, keepdims=True) + + return features + + +def extract_features_from_img_folder( + deployment: Deployment, images_folder_path: str, normalise_feats: bool = True +): + """ + Extract feature embeddings from a Geti deployment model for images in a folder + + :param deployment: The trained Geti deployment (model) to use for feature extraction. + :param images_folder_path: The path to the folder containing the images to extract features from. + :param normalise_feats: If True, the feature embeddings are normalised by dividing each feature + embedding vector by its respective 2nd-order vector norm (vector Euclidean norm) + :return: A numpy array containing the extracted feature embeddings of shape (n_images, feature_len) + """ + if not os.path.isdir(images_folder_path): + raise ValueError(f"img_folder {images_folder_path} is not a valid directory") + + images_in_folder = get_image_paths(images_folder_path) + sample_image = cv2.imread(list(images_in_folder.keys())[0]) + sample_image = cv2.cvtColor(sample_image, cv2.COLOR_BGR2RGB) + prediction = deployment.explain(sample_image) + feature_len = prediction.feature_vector.shape[0] + + features = np.zeros((len(images_in_folder.keys()), feature_len)) + + for k, id_image_path in tqdm( + enumerate(images_in_folder.keys()), + total=len(images_in_folder.keys()), + desc="Extracting features", + ): + numpy_image = cv2.imread(id_image_path) + numpy_rgb = cv2.cvtColor(numpy_image, cv2.COLOR_BGR2RGB) + prediction = deployment.explain(numpy_rgb) + features[k] = prediction.feature_vector + + if normalise_feats: + features = features / np.linalg.norm(features, axis=1, keepdims=True) + + return features + + +def get_grid_arrangement(n: int): + """ + Return the number of rows and columns for a grid arrangement of n items in a grid. + This function returns the grid arrangement with the closest number of rows and + columns to the square root of n. + """ + factors = [] + for current_factor in range(n): + if n % float(current_factor + 1) == 0: + factors.append(current_factor + 1) + + index_closest_to_sqrt = min( + range(len(factors)), key=lambda i: abs(factors[i] - sqrt(n)) + ) + + if factors[index_closest_to_sqrt] * factors[index_closest_to_sqrt] == n: + return factors[index_closest_to_sqrt], factors[index_closest_to_sqrt] else: - result = new_image - img = PILImage.fromarray(result) - display(img) + index_next = index_closest_to_sqrt + 1 + return factors[index_closest_to_sqrt], factors[index_next] + + +def display_sample_images_in_folder( + images_path: str, title: str = None, n_images: int = 9, show_labels: bool = True +): + """ + Display a random sample of images from a dataset folder + + :param images_path: path to the folder containing the images + :param title: title of the plot + :param n_images: number of images to display + :param show_labels: whether to show the name of the subfolder of the image as its title + """ + if not os.path.isdir(images_path): + raise ValueError(f"images_path {images_path} is not a valid directory") + + images_in_folder = get_image_paths(images_path) + random_indices = np.random.choice( + len(images_in_folder.keys()), n_images, replace=False + ) + n_rows, n_cols = get_grid_arrangement(n_images) + fig, axes = plt.subplots(n_rows, n_cols, figsize=(n_cols * 2, n_rows * 2)) + for i, ax in enumerate(axes.flatten()): + image = cv2.imread(list(images_in_folder.keys())[random_indices[i]]) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + image = cv2.resize(image, (480, 480)) + ax.imshow(image) + if show_labels: + ax.set_title( + list(images_in_folder.values())[random_indices[i]], + color="#0068b5", + fontsize=11, + wrap=True, + ) + ax.axis("off") + if title is None: + title = f"Sample images from {images_path}" + fig.suptitle(title, fontsize=16) + plt.tight_layout() + plt.show() diff --git a/notebooks/use_cases/utils/ood_detect.py b/notebooks/use_cases/utils/ood_detect.py new file mode 100644 index 00000000..5416a711 --- /dev/null +++ b/notebooks/use_cases/utils/ood_detect.py @@ -0,0 +1,67 @@ +# Copyright (C) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +import cv2 +import matplotlib.pyplot as plt +import numpy as np + +from .image import get_grid_arrangement, get_image_paths + + +def show_top_n_misclassifications( + images_dir: str, + scores: np.ndarray, + type_of_samples: str, + n_images: int = 9, +): + """ + Show top n misclassified images based on their OOD scores + (sorted by score in descending order for in-distribution samples and in ascending order for OOD samples). + :param images_dir: Path to directory with images. + :param scores: OOD scores for images. For in-distribution samples, the lower the score, the more OOD the sample is. + :param type_of_samples: Type of samples to show. Must be one of ['id', 'ood']. + :param n_images: Number of images to show. + + """ + images_paths_and_labels = get_image_paths(images_dir) + if type_of_samples == "id": + score_sort_inds = np.argsort(scores) + elif type_of_samples == "ood": + score_sort_inds = np.argsort(scores)[::-1] + else: + raise ValueError( + f"type_of_samples must be one of ['id', 'ood'], got {type_of_samples}" + ) + + images_paths = list(images_paths_and_labels.keys()) + image_paths_sorted_by_score = [images_paths[k] for k in score_sort_inds] + + n_rows, n_cols = get_grid_arrangement(n_images) + fig, axes = plt.subplots(n_rows, n_cols, figsize=(n_cols * 3, n_rows * 3)) + for i, ax in enumerate(axes.flatten()): + image = cv2.imread(image_paths_sorted_by_score[i]) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + image = cv2.resize(image, (480, 480)) + ax.imshow(image) + label = images_paths_and_labels[image_paths_sorted_by_score[i]] + ax.set_title( + f"Label: {label} \n (Sore : {scores[score_sort_inds[i]]:.2f})", + color="#0068b5", + fontsize=11, + wrap=True, + ) + ax.axis("off") + fig.suptitle(f"Top {n_images} misclassified {type_of_samples} images", fontsize=16) + plt.tight_layout() + plt.show() diff --git a/requirements/requirements-notebooks.txt b/requirements/requirements-notebooks.txt index 70aad6c3..82f48b17 100644 --- a/requirements/requirements-notebooks.txt +++ b/requirements/requirements-notebooks.txt @@ -3,3 +3,7 @@ jupyterlab>=3.5.3 jupyter-core>=4.11.2 ipywidgets>=8.0.0 mistune>=2.0.3 +imgaug>=0.4.0 +split-folders>=0.5.1 +scikit-learn>=0.24.2 +matplotlib>=3.6.0