Skip to content

Commit a3022d9

Browse files
spktrmspktrm
authored
Add files via upload
1 parent 1a5fc78 commit a3022d9

15 files changed

+1957
-0
lines changed

‎classification_report.py

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import numpy as np
2+
import tensorflow as tf
3+
import matplotlib.pyplot as plt
4+
import os
5+
import seaborn as sns
6+
import pandas as pd
7+
8+
from glob import glob
9+
from tensorflow import keras
10+
from sklearn.metrics import classification_report, confusion_matrix
11+
from sklearn.model_selection import train_test_split
12+
13+
X = pd.read_csv("handshapes/hand_keypoints.csv")
14+
y = np.array(X)[:, -1]
15+
X = np.array(X)[:, 1:-1]
16+
17+
model_path = "hand_models/modelv4.9"
18+
model = keras.models.load_model(model_path)
19+
20+
if os.path.exists(model_path + "/state.txt"):
21+
state = int(np.loadtxt(model_path + "/state.txt"))
22+
else:
23+
state = 1
24+
25+
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=state)
26+
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=state)
27+
28+
class_dir = "handshapes/*/"
29+
class_names = names = [os.path.basename(x[0:-1]) for x in glob(class_dir)]
30+
31+
model.summary()
32+
33+
# tf.keras.utils.plot_model(model, to_file="/" + model_path + "/model.png")
34+
35+
y_pred = np.argmax(model.predict(X_test), axis=1)
36+
37+
print(classification_report(y_test, y_pred, target_names=class_names))
38+
39+
cm = confusion_matrix(y_test, y_pred)
40+
41+
plt.figure()
42+
sns.heatmap(cm, xticklabels=class_names, yticklabels=class_names, annot=True, fmt='g')
43+
plt.xlabel('Prediction')
44+
plt.ylabel('Label')
45+
46+
# logs = pd.read_csv(model_path + "/logs.csv")
47+
# logs = np.array(logs)[:, 1]
48+
49+
# epochs_range = range(int(logs[0]))
50+
51+
# acc = logs[1].split(" ")
52+
# for i in range(acc.count("")):
53+
# acc.remove("")
54+
# for i in range(len(acc)):
55+
# acc[i] = acc[i].replace("[", "")
56+
# acc[i] = acc[i].replace("]", "")
57+
# acc[i] = float(acc[i])
58+
59+
# val_acc = logs[2].split(" ")
60+
# for i in range(val_acc.count("")):
61+
# val_acc.remove("")
62+
# for i in range(len(val_acc)):
63+
# val_acc[i] = val_acc[i].replace("[", "")
64+
# val_acc[i] = val_acc[i].replace("]", "")
65+
# val_acc[i] = float(val_acc[i])
66+
67+
# loss = logs[3].split(" ")
68+
# for i in range(loss.count("")):
69+
# loss.remove("")
70+
# for i in range(len(loss)):
71+
# loss[i] = loss[i].replace("[", "")
72+
# loss[i] = loss[i].replace("]", "")
73+
# loss[i] = float(loss[i])
74+
75+
# val_loss = logs[4].split(" ")
76+
# for i in range(val_loss.count("")):
77+
# val_loss.remove("")
78+
# for i in range(len(val_loss)):
79+
# val_loss[i] = val_loss[i].replace("[", "")
80+
# val_loss[i] = val_loss[i].replace("]", "")
81+
# val_loss[i] = float(val_loss[i])
82+
83+
# plt.figure()
84+
# plt.subplot(1, 2, 1)
85+
# plt.plot(epochs_range, acc, label='Training Accuracy')
86+
# plt.plot(epochs_range, val_acc, label='Validation Accuracy')
87+
# plt.legend(loc='lower right')
88+
# plt.title('Training and Validation Accuracy')
89+
90+
# plt.subplot(1, 2, 2)
91+
# plt.plot(epochs_range, loss, label='Training Loss')
92+
# plt.plot(epochs_range, val_loss, label='Validation Loss')
93+
# plt.legend(loc='upper right')
94+
# plt.title('Training and Validation Loss')
95+
plt.tight_layout()
96+
plt.show()
97+

‎create_data.py

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import cv2
2+
import mediapipe as mp
3+
import numpy as np
4+
import pandas as pd
5+
import os
6+
7+
from alive_progress import alive_bar
8+
from rotate import normalise, rotate_3d, subtract_offset
9+
10+
mp_drawing = mp.solutions.drawing_utils
11+
mp_hands = mp.solutions.hands
12+
13+
frame_count = sum(len(files) for _, _, files in os.walk("handshapes"))
14+
15+
hand = []
16+
17+
model = mp_hands.Hands(min_detection_confidence=0.75, min_tracking_confidence=0.75, max_num_hands=2)
18+
19+
labels = os.listdir("handshapes")
20+
label_idx = -1
21+
n = 0
22+
with alive_bar(frame_count - 1) as bar:
23+
for root, dirs, files in os.walk("handshapes"):
24+
for f in files:
25+
if f.endswith(".jpg"):
26+
image = cv2.imread(os.path.join(root, f))
27+
28+
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
29+
image.flags.writeable = False
30+
results = model.process(image)
31+
image.flags.writeable = True
32+
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
33+
34+
if results.multi_hand_landmarks is not None:
35+
for hand_landmarks_idx in range(len(results.multi_hand_landmarks)):
36+
hand_landmarks = results.multi_hand_landmarks[hand_landmarks_idx]
37+
38+
keypoints = results.multi_hand_landmarks[hand_landmarks_idx].landmark._values
39+
kp = []
40+
for point in keypoints:
41+
kp.append(np.array([point.x, point.y, point.z]))
42+
43+
label = results.multi_handedness[0].classification._values[0].label
44+
45+
keypoints = np.array(kp)
46+
keypoints = subtract_offset(keypoints)
47+
keypoints = rotate_3d(keypoints)
48+
keypoints = normalise(keypoints)
49+
50+
if label == 'Right':
51+
hand.append(np.append(keypoints.flatten(), label_idx))
52+
n += 1
53+
else:
54+
keypoints[:, 0] *= -1
55+
hand.append(np.append(keypoints.flatten(), label_idx))
56+
n += 1
57+
58+
bar()
59+
60+
label_idx += 1
61+
62+
print("\nAverage {} hands/frame\n".format(round(n/frame_count, 2)))
63+
64+
hand_df = pd.DataFrame(np.array(hand))
65+
hand_df.to_csv("handshapes/hand_keypoints.csv")

‎create_data_for_clustering.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import cv2
2+
import mediapipe as mp
3+
import numpy as np
4+
import pandas as pd
5+
import os
6+
7+
from alive_progress import alive_bar
8+
from rotate import normalise, rotate_3d, subtract_offset
9+
10+
mp_drawing = mp.solutions.drawing_utils
11+
mp_hands = mp.solutions.hands
12+
13+
file = "WIN_20210422_17_16_38_Pro"
14+
15+
frame_count = sum(len(files) for _, _, files in os.walk(file + "/frames"))
16+
17+
right_hand = []
18+
right_hand_paths = []
19+
left_hand = []
20+
left_hand_paths = []
21+
22+
model = mp_hands.Hands(min_detection_confidence=0.75, min_tracking_confidence=0.75, max_num_hands=2)
23+
24+
labels = os.listdir(file + "/frames")
25+
n = 0
26+
with alive_bar(frame_count - 1) as bar:
27+
for root, dirs, files in os.walk(file + "/frames"):
28+
for f in files:
29+
if f.endswith(".jpg"):
30+
image = cv2.imread(os.path.join(root, f))
31+
32+
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
33+
image.flags.writeable = False
34+
results = model.process(image)
35+
image.flags.writeable = True
36+
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
37+
38+
if results.multi_hand_landmarks is not None:
39+
for hand_landmarks_idx in range(len(results.multi_hand_landmarks)):
40+
hand_landmarks = results.multi_hand_landmarks[hand_landmarks_idx]
41+
42+
keypoints = results.multi_hand_landmarks[hand_landmarks_idx].landmark._values
43+
kp = []
44+
for point in keypoints:
45+
kp.append(np.array([point.x, point.y, point.z]))
46+
47+
label = results.multi_handedness[0].classification._values[0].label
48+
49+
keypoints = np.array(kp)
50+
# keypoints = subtract_offset(keypoints)
51+
# keypoints = rotate_3d(keypoints)
52+
# keypoints = normalise(keypoints)
53+
54+
if label == 'Right':
55+
right_hand.append(keypoints.flatten())
56+
right_hand_paths.append(os.path.join(root, f))
57+
n += 1
58+
else:
59+
left_hand.append(keypoints.flatten())
60+
left_hand_paths.append(os.path.join(root, f))
61+
n += 1
62+
63+
bar()
64+
65+
print("\nAverage {} hands/frame\n".format(round(n/frame_count, 2)))
66+
67+
right_hand_df = pd.DataFrame(np.array(right_hand))
68+
left_hand_df = pd.DataFrame(np.array(left_hand))
69+
70+
right_hand_paths = pd.DataFrame(np.array(right_hand_paths))
71+
left_hand_paths = pd.DataFrame(np.array(left_hand_paths))
72+
73+
right_hand_df.to_csv(file + "/right_hand_keypoints.csv")
74+
left_hand_df.to_csv(file + "/left_hand_keypoints.csv")
75+
76+
right_hand_paths.to_csv(file + "/right_hand_paths.csv")
77+
left_hand_paths.to_csv(file + "/left_hand_paths.csv")

‎dbscan_clustering.py

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import numpy as np
2+
import pandas as pd
3+
import matplotlib.pyplot as plt
4+
import os
5+
import cv2
6+
7+
from alive_progress import alive_bar
8+
from sklearn.decomposition import PCA
9+
from sklearn.cluster import DBSCAN
10+
from rotate import normalise, rotate_3d
11+
12+
# pca params
13+
n_PCA = 15
14+
eps = 0.1
15+
16+
# file = "Basic words - Auslan"
17+
# file = "Auslan COVID-19 Update - 29 Jan 2021"
18+
# file = "WIN_20210418_10_11_51_Pro"
19+
file = "WIN_20210422_17_16_38_Pro"
20+
21+
if not os.path.exists(file + "/dbscan_right_hand_clusters"):
22+
os.mkdir(file + "/dbscan_right_hand_clusters")
23+
24+
print("\nloading hand csv file...")
25+
df = pd.read_csv(file + "/right_hand_keypoints.csv")
26+
df = np.array(df)
27+
df = df[:, 1:]
28+
for i in range(len(df)):
29+
df[i][0::3] -= df[i][0]
30+
df[i][1::3] -= df[i][1]
31+
df[i][2::3] -= df[i][2]
32+
33+
pts = np.vstack([df[i][0::3], df[i][1::3], df[i][2::3]]).T
34+
pts = normalise(pts)
35+
36+
df[i] = rotate_3d(pts).flatten()
37+
38+
paths = pd.read_csv(file + "/right_hand_paths.csv")
39+
paths = np.array(paths)
40+
paths = paths[:, 1:]
41+
paths = paths.astype('str')
42+
print("hand csv file loaded!\n")
43+
44+
n_components = min(n_PCA, min(df.shape[0], df.shape[1]))
45+
print("performing PCA on hands for first {} components...".format(n_components))
46+
pcaf = PCA(n_components=n_components)
47+
df = pcaf.fit_transform(df)
48+
var = round(np.sum(pcaf.explained_variance_ratio_[:n_components]) * 100, 2)
49+
print(("{}"+'%'+" of variance explained with first {} components.").format(var, n_components))
50+
print("PCA done.\n")
51+
52+
print("performing dbscan clustering on hand data...")
53+
df = pcaf.fit_transform(df)
54+
dbscan = DBSCAN(eps=eps, n_jobs=-1, metric='cosine')
55+
dbscan.fit(df)
56+
labels = dbscan.fit_predict(df)
57+
print("predicted no clusters: {}\n".format(len(np.unique(labels)[1:])))
58+
59+
with alive_bar(len(np.delete(labels, np.where(labels == -1)))) as bar:
60+
61+
for i in range(len(np.unique(labels)[1:])):
62+
cluster_folder_path = file + "/dbscan_right_hand_clusters/" + str(i)
63+
64+
if not os.path.exists(cluster_folder_path):
65+
os.mkdir(cluster_folder_path)
66+
67+
i_labels = np.where(labels == i)[0]
68+
69+
for j in i_labels:
70+
image = cv2.imread(paths[j][0])
71+
no = np.where(np.char.find(paths, paths[j][0].split('/')[-1]) != -1)[0][0]
72+
cv2.imwrite(cluster_folder_path + '/' + str(no) + ".jpg", image)
73+
bar()
74+
75+
if not os.path.exists(file + "/dbscan_left_hand_clusters"):
76+
os.mkdir(file + "/dbscan_left_hand_clusters")
77+
78+
print("\nloading hand csv file...")
79+
df = pd.read_csv(file + "/left_hand_keypoints.csv")
80+
df = np.array(df)
81+
df = df[:, 1:]
82+
for i in range(len(df)):
83+
df[i][0::3] -= df[i][0]
84+
df[i][1::3] -= df[i][1]
85+
df[i][2::3] -= df[i][2]
86+
df[i] = rotate_3d(np.vstack([df[i][0::3], df[i][1::3], df[i][2::3]]).T).flatten()
87+
# df = df[:, 3:]
88+
89+
paths = pd.read_csv(file + "/left_hand_paths.csv")
90+
paths = np.array(paths)
91+
paths = paths[:, 1:]
92+
paths = paths.astype('str')
93+
print("hand csv file loaded!\n")
94+
95+
n_components = min(n_PCA, min(df.shape[0], df.shape[1]))
96+
print("performing PCA on hands for first {} components...".format(n_components))
97+
pcaf = PCA(n_components=n_components)
98+
df = pcaf.fit_transform(df)
99+
var = round(np.sum(pcaf.explained_variance_ratio_[:n_components]) * 100, 2)
100+
print(("{}"+'%'+" of variance explained with first {} components.").format(var, n_components))
101+
print("PCA done.\n")
102+
103+
print("performing dbscan clustering on hand data...")
104+
df = pcaf.fit_transform(df)
105+
dbscan = DBSCAN(eps=eps, n_jobs=-1, metric='cosine')
106+
dbscan.fit(df)
107+
labels = dbscan.fit_predict(df)
108+
print("predicted no clusters: {}\n".format(len(np.unique(labels)[1:])))
109+
110+
with alive_bar(len(np.delete(labels, np.where(labels == -1)))) as bar:
111+
112+
for i in range(len(np.unique(labels)[1:])):
113+
cluster_folder_path = file + "/dbscan_left_hand_clusters/" + str(i)
114+
115+
if not os.path.exists(cluster_folder_path):
116+
os.mkdir(cluster_folder_path)
117+
118+
i_labels = np.where(labels == i)[0]
119+
120+
for j in i_labels:
121+
image = cv2.imread(paths[j][0])
122+
no = np.where(np.char.find(paths, paths[j][0].split('/')[-1]) != -1)[0][0]
123+
cv2.imwrite(cluster_folder_path + '/' + str(no) + ".jpg", image)
124+
bar()

0 commit comments

Comments
 (0)