[12601] model integration testing

.gitignore

@@ -1,9 +1,11 @@
+test_status.json
+
 \.idea/
 \.vscode
 clearcut_detection_backend/clearcuts/migrations/
 clearcut_detection_backend/model/predicted/
 logs.txt
-data/
+clearcut_detection_backend/data/
 logs/
 venv/
 _install/
@@ -80,4 +82,4 @@ credentials.json
 # External files
 landcover.zip
 docker-compose-django-debug.yml
-dyman_settings.py
+dyman_settings.py

clearcut_detection_backend/docker-compose-test.yml

@@ -0,0 +1,31 @@
+version: '3'
+services:
+    model:
+        build:
+            context: ./model
+            dockerfile: model.Dockerfile
+        image: clearcut_detection/model
+        env_file:
+            - ./model/model.env
+        volumes:
+            - ./model/:/model
+            - ./data/:/model/data
+        working_dir: /model
+        environment:
+            - CUDA_VISIBLE_DEVICES=0
+        ports:
+            - '5000:5000'
+        command: /bin/bash -c "python3 app.py"
+
+    test:
+        build:
+            context: ./
+            dockerfile: test.Dockerfile
+        image: clearcut_detection/test
+        volumes:
+            - ./:/code
+        working_dir: /code
+        command: /bin/bash -c "pip install -r ./test/requirements.txt && python test.py"
+
+volumes:
+  data:

clearcut_detection_backend/model/model.Dockerfile

@@ -1,11 +1,13 @@
-FROM python:3.6
+FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04
+
+RUN apt-get update && apt-get install -y python3-pip
 
 RUN mkdir /model
 
 WORKDIR /model
 
-ADD requirements.txt /model
+COPY requirements.txt /model
 
-RUN pip install -r requirements.txt
+RUN pip3 install -r requirements.txt
 
-ADD . /model/
+COPY . /model/

clearcut_detection_backend/model/predict_raster.py

@@ -31,6 +31,8 @@
 import warnings
 warnings.filterwarnings('ignore')
 
+os.environ.get('CUDA_VISIBLE_DEVICES', '0')
+
 logging.basicConfig(format='%(asctime)s %(message)s')
 
 def load_model(network, model_weights_path, channels, neighbours):

clearcut_detection_backend/model/requirements.txt

@@ -1,6 +1,6 @@
 catalyst==19.5
 Flask==1.1.1
-geopandas==0.5.1
+geopandas==0.8.1
 google-api-python-client==1.8.0
 google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
@@ -14,3 +14,4 @@ tqdm==4.19.9
 oauth2client==4.1.3
 Pillow==6.2.2
 PyYAML==5.1.1
+opencv-python-headless==4.1.0.25

clearcut_detection_backend/test.Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.6
+
+RUN mkdir /test
+WORKDIR /test
+
+COPY ./test/requirements.txt /test
+RUN pip install -r requirements.txt
+
+RUN apt-get update -y && apt-get install -y \
+    software-properties-common
+
+RUN add-apt-repository -r ppa:ubuntugis/ppa && apt-get update
+RUN apt-get update
+RUN apt-get install gdal-bin -y
+RUN apt-get install libgdal-dev -y
+RUN export CPLUS_INCLUDE_PATH=/usr/include/gdal
+RUN export C_INCLUDE_PATH=/usr/include/gdal
+RUN pip install GDAL==$(gdal-config --version | awk -F'[.]' '{print $1"."$2}')
+
+ADD . /test/

clearcut_detection_backend/test.py

@@ -0,0 +1,5 @@
+from test.predict import model_predict
+from test.evaluation import model_evaluate
+
+results, test_tile_path = model_predict()
+model_evaluate(results, test_tile_path)

clearcut_detection_backend/test/evaluation.py

@@ -0,0 +1,141 @@
+import os
+import json
+import yaml
+
+import numpy as np
+import pandas as pd
+import rasterio
+import geopandas
+
+from tqdm import tqdm
+from rasterio import features
+
+from test.polygon_metrics import f1_score_evaluation, polygonize
+from test.utils import GOLD_DICE, GOLD_F1SCORE, GOLD_IOU, SUCCESS_THRESHOLD, IOU_THRESHOLD
+from test.test_data_prepare import get_gt_polygons
+
+def dice_coef(y_true, y_pred, eps=1e-7):
+    y_true_f = y_true.flatten()
+    y_pred_f = y_pred.flatten()
+    intersection = np.sum(y_true_f * y_pred_f)
+    return (2. * intersection + eps) / (np.sum(y_true_f) + np.sum(y_pred_f) + eps)
+
+
+def iou(y_true, y_pred, smooth=1.0):
+    y_true_f = y_true.flatten()
+    y_pred_f = y_pred.flatten()
+    intersection = np.sum(y_true_f * y_pred_f)
+    return (1. * intersection + smooth) / (np.sum(y_true_f) + np.sum(y_pred_f) - intersection + smooth)
+
+
+def confusion_matrix(y_true, y_pred):
+    mm, mn, nm, nn = 0, 0, 0, 0
+    M, N = 0, 0
+    for i in range(len(y_true)):
+        if(y_true.iloc[i] == y_pred.iloc[i]):
+            if(y_true.iloc[i] == 1):
+                M += 1
+                mm += 1
+            else:
+                N += 1
+                nn += 1
+        else:
+            if(y_true.iloc[i] == 1):
+                M += 1
+                mn += 1
+            else:
+                N += 1
+                nm += 1
+    return mm, mn, nm, nn, M, N
+
+
+def get_raster_masks(reference_tif, model_result):
+    raster = {}
+    with rasterio.open(reference_tif) as src:
+        filenames = {}
+        filenames['mask'] = get_gt_polygons()
+        filenames['predicted'] = os.path.join('data', model_result[0].get('polygons'))
+        for name in filenames:
+            gt_polygons = geopandas.read_file(filenames[name])
+            gt_polygons = gt_polygons.to_crs(src.crs)
+            raster[name] = features.rasterize(shapes=gt_polygons['geometry'],
+                                              out_shape=(src.height, src.width),
+                                              transform=src.transform,
+                                              default_value=1)
+
+    return raster
+
+def load_config():
+    with open('./model/predict_config.yml', 'r') as config:
+        cfg = yaml.load(config, Loader=yaml.SafeLoader)
+
+    models = cfg['models']
+    save_path = cfg['prediction']['save_path']
+    threshold = cfg['prediction']['threshold']
+    input_size = cfg['prediction']['input_size']
+
+    return models, save_path, threshold, input_size
+
+
+def evaluate(model_result, test_tile_path):
+    raster = get_raster_masks(test_tile_path['current'], model_result)
+    _, _, _, size = load_config()
+
+    res_cols = ['name', 'dice_score', 'iou_score', 'pixel_amount']
+    test_df_results = pd.DataFrame(columns=res_cols)
+    dices, ious = [], []
+    test_polys, truth_polys = [], []
+    for i in tqdm(range(raster['mask'].shape[0] // size)):
+        for j in range(raster['mask'].shape[1] // size):
+            instance_name = f'{i}_{j}'
+            mask = raster['mask'][i*size : (i+1)*size, j*size : (j+1)*size]
+            if mask.sum() > 0:
+                prediction = raster['predicted'][i*size : (i+1)*size, j*size : (j+1)*size]
+                test_polys.append(polygonize(prediction.astype(np.uint8)))
+                truth_polys.append(polygonize(mask.astype(np.uint8)))
+
+                dice_score = dice_coef(mask, prediction)
+                iou_score = iou(mask, prediction, smooth=1.0)
+
+                dices.append(dice_score)
+                ious.append(iou_score)
+
+                pixel_amount = mask.sum()
+
+                test_df_results = test_df_results.append({'name': instance_name,
+                                        'dice_score': dice_score, 'iou_score': iou_score, 'pixel_amount': pixel_amount}, ignore_index=True)
+
+    log = pd.DataFrame(columns=['f1_score', 'threshold', 'TP', 'FP', 'FN'])
+    for threshold in np.arange(0.1, 1, 0.1):
+        F1score, true_pos_count, false_pos_count, false_neg_count, total_count = f1_score_evaluation(test_polys, truth_polys, threshold=threshold)
+        log = log.append({'f1_score': round(F1score,4),
+                          'threshold': round(threshold,2),
+                          'TP':int(true_pos_count),
+                          'FP':int(false_pos_count),
+                          'FN':int(false_neg_count)}, ignore_index=True)
+
+    return log, np.average(dices), np.average(ious)
+
+
+
+def model_evaluate(model_result, test_tile_path):
+    f1_score_test, dice, iou = evaluate(model_result, test_tile_path)
+
+    f1_score_test = f1_score_test[f1_score_test['threshold'] == IOU_THRESHOLD]['f1_score'].to_numpy()
+    f1_score_standard = GOLD_F1SCORE
+
+    result = {}
+    result['f1_score'] = float(f1_score_standard - f1_score_test[0])
+    result['dice_score'] = GOLD_DICE - dice
+    result['iou_score'] = GOLD_IOU - iou
+    result['status'] = (result['f1_score'] < SUCCESS_THRESHOLD) \
+                     & (result['dice_score'] < SUCCESS_THRESHOLD) \
+                     & (result['iou_score'] < SUCCESS_THRESHOLD)
+
+    if result['status']:
+        result['status'] = str(result['status']).replace('True', 'success')
+    else:
+        result['status'] = str(result['status']).replace('False', 'failed')
+
+    with open('test_status.json', 'w') as outfile:
+        json.dump(result, outfile)

clearcut_detection_backend/test/polygon_metrics.py

@@ -0,0 +1,117 @@
+import os
+import cv2
+import numpy as np
+
+from scipy import ndimage as ndi
+from shapely.geometry import Polygon
+from skimage.segmentation import watershed
+from skimage.feature import peak_local_max
+
+import matplotlib.pyplot as plt
+
+def watershed_segmentation(image):
+	distance = ndi.distance_transform_edt(image)
+	local_maxi = peak_local_max(distance, indices=False, footprint=np.ones((3, 3)),
+	                            labels=image)
+	markers = ndi.label(local_maxi)[0]
+	labels = watershed(-distance, markers, mask=image)
+	return labels, distance
+
+def polygonize(raster_array, meta=None, transform=False):
+    contours, hierarchy = cv2.findContours(raster_array.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+    polygons = []
+    for i in range(len(contours)):
+        c = contours[i]
+        n_s = (c.shape[0], c.shape[2])
+        if n_s[0] > 2:
+            if transform:
+                polys = [tuple(i) * meta['transform'] for i in c.reshape(n_s)]
+            else:
+                polys = [tuple(i) for i in c.reshape(n_s)]
+            polygons.append(Polygon(polys))
+    return polygons
+
+def iou_poly(test_poly, truth_poly):
+    iou_score = 0
+    intersection_result = test_poly.intersection(truth_poly)
+    if not intersection_result.is_valid:
+        intersection_result = intersection_result.buffer(0)
+    if not intersection_result.is_empty:
+        intersection_area = intersection_result.area
+        union_area = test_poly.union(truth_poly).area
+        iou_score = intersection_area / union_area
+    else:
+        iou_score = 0
+    return iou_score
+
+
+def score(test_polys, truth_polys, threshold=0.5):
+    true_pos_count = 0
+    true_neg_count = 0
+    false_pos_count = 0
+    false_neg_count = 0
+    total_count = 0
+    for test_poly, truth_poly in zip(test_polys, truth_polys):
+        if len(test_poly)==0 and len(truth_poly)==0:
+            true_neg_count += 1
+            total_count+=1
+        elif len(test_poly)==0 and len(truth_poly)>0:
+            false_pos_count += 1
+            total_count+=1
+        elif len(test_poly)>0 and len(truth_poly)==0:
+            false_neg_count += 1
+            total_count+=1
+        else:
+            intersected=[]
+
+            for test_p in test_poly:
+                for truth_p in truth_poly:
+                    if not test_p.is_valid:
+                        test_p = test_p.buffer(0)
+                    if not truth_p.is_valid:
+                        truth_p = truth_p.buffer(0)
+                    if test_p.intersection(truth_p).is_valid:
+                        if not test_p.intersection(truth_p).is_empty:
+                            intersected.append([test_p, truth_p])
+
+            if len(intersected) < len(test_poly):
+                false_neg_count += (len(test_poly) - len(intersected))
+                total_count+=(len(test_poly) - len(intersected))
+            if len(intersected) < len(truth_poly):
+                false_pos_count += (len(truth_poly) - len(intersected))
+                total_count+=(len(truth_poly) - len(intersected))
+            for inter in intersected:
+                iou_score = iou_poly(inter[0], inter[1])
+
+                if iou_score >= threshold:
+                    true_pos_count += 1
+                    total_count+=1
+                else:
+                    false_pos_count += 1
+                    total_count+=1
+    return true_pos_count, false_pos_count, false_neg_count, total_count
+
+
+def f1_score_evaluation(test_polys, truth_polys, threshold = 0.5):
+    if len(truth_polys)==0 and len(test_polys)!=0:
+        true_pos_count = 0
+        false_pos_count = len(test_polys)
+        false_neg_count = 0
+        total_count = len(test_polys)
+    elif len(truth_polys)==0 and len(test_polys)==0:
+        true_pos_count = len(test_polys)
+        false_pos_count = 0
+        false_neg_count = 0
+        total_count = len(test_polys)
+    else:
+        true_pos_count, false_pos_count, false_neg_count, total_count = score(test_polys, truth_polys,
+                                                                 threshold=threshold
+                                                                 )
+
+    if (true_pos_count > 0):
+        precision = float(true_pos_count) / (float(true_pos_count) + float(false_pos_count))
+        recall = float(true_pos_count) / (float(true_pos_count) + float(false_neg_count))
+        F1score = 2.0 * precision * recall / (precision + recall)
+    else:
+        F1score = 0
+    return F1score, true_pos_count, false_pos_count, false_neg_count, total_count