Basic version to compare embeddings and the last level opf prediction

predictions/plotting.py

@@ -0,0 +1,86 @@
+import json
+
+from matplotlib import pyplot as plt
+
+
+def load_results():
+    with open("predictions/test_results/knn.json", 'r') as f:
+        results = json.load(f)
+    return results
+
+def plot_all():
+    results = load_results()
+    print(f"average elapsed time to detect a sign: {get_general_elapsed_time(results)}")
+    plot_general_accuracy(results)
+    for label in results.keys():
+        plot_accuracy_per_label(results, label)
+
+
+def general_accuracy(results):
+    label_accuracy = get_label_accuracy(results)
+    accuracy = []
+    amount = []
+    response = []
+    for label in label_accuracy.keys():
+        for index, value in enumerate(label_accuracy[label]):
+            if index >= len(accuracy):
+                accuracy.append(0)
+                amount.append(0)
+            accuracy[index] += label_accuracy[label][index]
+            amount[index] += 1
+    for a, b in zip(accuracy, amount):
+        if b < 5:
+            break
+        response.append(a / b)
+    return response
+def plot_general_accuracy(results):
+    accuracy = general_accuracy(results)
+    plt.plot(accuracy)
+    plt.title = "General accuracy"
+    plt.ylabel('accuracy')
+    plt.xlabel('buffer')
+    plt.show()
+
+
+def plot_accuracy_per_label(results, label):
+    accuracy = get_label_accuracy(results)
+    plt.plot(accuracy[label], label=label)
+    plt.titel = f"Accuracy per label {label}"
+    plt.ylabel('accuracy')
+    plt.xlabel('prediction')
+    plt.legend()
+    plt.show()
+
+def get_label_accuracy(results):
+    accuracy = {}
+    amount = {}
+    response = {}
+    for label, predictions in results.items():
+        if label not in accuracy:
+            accuracy[label] = []
+            amount[label] = []
+        for prediction in predictions:
+            for index, value in enumerate(prediction["predictions"]):
+                if index >= len(accuracy[label]):
+                    accuracy[label].append(0)
+                    amount[label].append(0)
+                accuracy[label][index] += 1 if value["correct"] else 0
+                amount[label][index] += 1
+    for label in accuracy:
+        response[label] = []
+        for index, value in enumerate(accuracy[label]):
+            if amount[label][index] < 2:
+                break
+            response[label].append(accuracy[label][index] / amount[label][index])
+    return response
+
+def get_general_elapsed_time(results):
+    label_time = get_label_elapsed_time(results)
+    return sum([label_time[label] for label in results]) / len(results)
+
+def get_label_elapsed_time(results):
+    return {label: sum([result["elapsed_time"] for result in results[label]]) / len(results[label]) for label in results}
+
+
+if __name__ == '__main__':
+    plot_all()

predictions/predictor.py

@@ -222,6 +222,18 @@ class Predictor:
 
         return keypoints
 
+
+    def get_embedding(self, keypoints):
+        # run model on frame
+        self.model.eval()
+        with torch.no_grad():
+            keypoints = torch.from_numpy(np.array([keypoints])).float().to(device)
+            new_embeddings = self.model(keypoints).cpu().numpy().tolist()[0]
+        return new_embeddings
+
+    def predict(self, embeddings):
+        return self.predictor.predict(embeddings)
+
     def make_prediction(self, keypoints):
         # run model on frame
         self.model.eval()
@@ -230,3 +242,26 @@ class Predictor:
             new_embeddings = self.model(keypoints).cpu().numpy().tolist()[0]
 
         return self.predictor.predict(new_embeddings)
+
+    def validation(self):
+        # load validation data
+        validation_data = np.load('validation_data.npy', allow_pickle=True)
+        validation_labels = np.load('validation_labels.npy', allow_pickle=True)
+
+        # run model on validation data
+        self.model.eval()
+        with torch.no_grad():
+            validation_embeddings = self.model(torch.from_numpy(validation_data).float().to(device)).cpu().numpy()
+
+        # predict validation data
+        predictions = self.predictor.predict(validation_embeddings)
+
+        # calculate accuracy
+        correct = 0
+        for i in range(len(predictions)):
+            if predictions[i] == validation_labels[i]:
+                correct += 1
+        accuracy = correct / len(predictions)
+        print('Accuracy: ' + str(accuracy))
+
+

predictions/svm_model.py

@@ -19,7 +19,6 @@ class SVM:
         self.label_name_to_label = df[["label_name", "labels"]]
         self.label_name_to_label.columns = ["label_name", "label"]
         self.label_name_to_label = self.label_name_to_label.drop_duplicates()
-        print(self.label_name_to_label)
 
         self.train()
 
@@ -32,8 +31,4 @@ class SVM:
         score = self.clf.predict_log_proba(key_points_embeddings)
         # TODO fix dictionary
         label = label.item()
-        print("test")
-        print(self.label_name_to_label.loc[self.label_name_to_label["label"] == label]["label_name"].iloc[0])
-        print("test2")
-        print(score)
         return self.label_name_to_label.loc[self.label_name_to_label["label"] == label]["label_name"].iloc[0], score[0][label]

predictions/validation.py

@@ -0,0 +1,137 @@
+import json
+import os
+import time
+
+import cv2
+import numpy as np
+from matplotlib import pyplot as plt
+
+from predictions.k_nearest import KNearestNeighbours
+from predictions.predictor import Predictor
+from predictions.svm_model import SVM
+
+buffer_size = 15
+
+
+def predict_video(predictor, path_video):
+    # open mp4 video
+    cap = cv2.VideoCapture(path_video)
+    buffer = []
+    ret, img = cap.read()  # read one frame from the 'capture' object; img is (H, W, C)
+    desired_fps = 15
+    original_fps = int(cap.get(cv2.CAP_PROP_FPS))
+    print("Original FPS: ", original_fps)
+    # Calculate the frame skipping rate based on desired frame rate
+    frame_skip = original_fps // desired_fps
+    if frame_skip == 0:
+        frame_skip = 1
+    print("Frame skip: ", frame_skip)
+    frame_number = 0
+    while img is not None:
+        pose = predictor.extract_keypoints(img)
+        if pose is not None and frame_number % frame_skip == 0:
+            buffer.append(pose)
+        frame_number += 1
+        ret, img = cap.read()  # read one frame from the 'capture' object; img is (H, W, C)
+    print(len(buffer))
+    return buffer
+
+
+def get_embeddings(predictor, buffer, name):
+    # check if file exists with name
+    # if os.path.exists("predictions/test_embeddings/" + name + ".csv"):
+    #     print("Loading embeddings from file")
+    #     # load embeddings from file
+    #     with open("predictions/test_embeddings/" + name + ".csv", 'r') as f:
+    #         embeddings = json.load(f)
+    # else:
+    embeddings = []
+    for index in range(buffer_size, len(buffer)):
+        embedding = predictor.get_embedding(buffer[index - buffer_size:index])
+        embeddings.append(embedding)
+    with open("predictions/test_embeddings/" + name + ".csv", 'w') as f:
+        json.dump(embeddings, f)
+    return embeddings
+
+
+def compare_embeddings(predictor, embeddings, label_video, ):
+    results = []
+    for embedding in embeddings:
+        label, score = predictor.predict(embedding)
+
+        results.append({"label": label, "score": score, "label_video": label_video, "correct": label == label_video})
+    return results
+
+
+def predict_video_files(predictor, path_video, label_video):
+    buffer = predict_video(predictor, path_video)
+    embeddings = get_embeddings(predictor, buffer, path_video.split("/")[-1].split(".")[0])
+    return compare_embeddings(predictor, embeddings, label_video)
+
+
+def get_test_data(data_folder):
+    files = np.array([data_folder + f for f in os.listdir(data_folder) if f.endswith(".mp4")])
+    train_test = [f.split("/")[-1].split("!")[1] for f in files]
+    test_files = files[np.array(train_test) == "test"]
+    test_labels = [f.split("/")[-1].split("!")[0] for f in test_files]
+
+    return test_files, test_labels
+
+
+def test_data(predictor, data_folder):
+    results = {}
+    for path_video, label_video in zip(*get_test_data(data_folder)):
+        print(path_video, label_video)
+        start_time = time.time()
+        prediction = predict_video_files(predictor, path_video, label_video)
+        end_time = time.time()
+        elapsed_time = end_time - start_time
+
+        # divide elapsed time by amount of predictions made so it represents an avarage execution time
+        if len(prediction) > 0:
+            elapsed_time /= len(prediction)
+        if label_video not in results:
+            results[label_video] = []
+        results[label_video].append({"predictions": prediction, "elapsed_time": elapsed_time, "video": path_video})
+
+        print("DONE")
+    return results
+
+
+def plot_general_accuracy(results):
+    accuracy = []
+    amount = []
+    for result in results:
+        for index, value in enumerate(result[0]):
+            if len(accuracy) <= index:
+                accuracy.append(0)
+                amount.append(0)
+            accuracy[index] += 1 if value["correct"] else 0
+            amount[index] += 1
+    # plot the general accuracy
+    plt.plot(accuracy)
+    plt.show()
+
+
+if __name__ == "__main__":
+    type_predictor = "knn"
+    if type_predictor == "knn":
+        k = 1
+        predictor_type = KNearestNeighbours(k)
+    elif type_predictor == "svm":
+        predictor_type = SVM()
+    else:
+        predictor_type = KNearestNeighbours(1)
+
+    # embeddings_path = 'embeddings/basic-signs/embeddings.csv'
+    embeddings_path = 'embeddings/fingerspelling/embeddings.csv'
+
+    predictor = Predictor(embeddings_path, predictor_type)
+
+    data_folder = '/home/tibe/Projects/design_project/sign-predictor/data/fingerspelling/data/'
+    results = test_data(predictor, data_folder)
+    # write results to a results json file
+    with open("predictions/test_results/" + type_predictor + ".json", 'w') as f:
+        json.dump(results, f)
+    print(results)
+    # plot_general_accuracy(results)