Resolve WES-90 "Integrate signpredictor in courses"

Assets/MediaPipeUnity/Common/Scripts/KeypointManager.cs

@@ -0,0 +1,147 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using System.Linq;
+
+public class ModelInfo
+{
+    public List<int> pose_landmarks;
+    public List<int> hand_landmarks;
+}
+
+public class KeypointManager
+{
+
+    private ModelInfo model_info;
+    private List<List<float>> keypoints_buffer;
+
+    public KeypointManager()
+    {
+        TextAsset model_info_json = Resources.Load<TextAsset>("Models/FingerSpelling/landmarks");
+        this.model_info = JsonUtility.FromJson<ModelInfo>(model_info_json.text);
+        this.keypoints_buffer = new List<List<float>>();
+    }
+
+    private (List<float>, List<float>) normalizeHand(List<float> hand_x, List<float> hand_y)
+    {
+
+        float min_x = hand_x.Min();
+        float min_y = hand_y.Min();
+
+        float max_x = hand_x.Max();
+        float max_y = hand_y.Max();
+
+        float width = max_x - min_x;
+        float height = max_y - min_y;
+
+        if (width == 0 || height == 0)
+        {
+            return (hand_x, hand_y);
+        }
+
+        float center_x = (min_x + max_x) / 2;
+        float center_y = (min_y + max_y) / 2;
+
+        List<float> normalized_x = new List<float>();
+        List<float> normalized_y = new List<float>();
+
+        for (int i = 0; i < hand_x.Count; i++)
+        {
+            normalized_x.Add((hand_x[i] - center_x) / width);
+            normalized_y.Add((hand_y[i] - center_y) / height);
+        }
+
+        return (normalized_x, normalized_y);
+    }
+
+    public void addLandmarks(Mediapipe.NormalizedLandmarkList poseLandmarks, Mediapipe.NormalizedLandmarkList leftHandLandmarks, Mediapipe.NormalizedLandmarkList rightHandLandmarks)
+    {
+        List<float> pose_x = new List<float>();
+        List<float> pose_y = new List<float>();
+        List<float> left_hand_x = new List<float>();
+        List<float> left_hand_y = new List<float>();
+        List<float> right_hand_x = new List<float>();
+        List<float> right_hand_y = new List<float>();
+
+        if (poseLandmarks != null)
+        {
+            foreach (var landmark_index in model_info.pose_landmarks)
+            {
+                pose_x.Add(poseLandmarks.Landmark[landmark_index].X);
+                pose_y.Add(poseLandmarks.Landmark[landmark_index].Y);
+            }
+        }
+        else
+        {
+            foreach (var landmark_index in model_info.pose_landmarks)
+            {
+                pose_x.Add(0);
+                pose_y.Add(0);
+            }
+        }
+
+
+
+        foreach (var landmark_index in model_info.hand_landmarks)
+        {
+            if (leftHandLandmarks == null)
+            {
+                left_hand_x.Add(0);
+                left_hand_y.Add(0);
+            }
+            else
+            {
+                left_hand_x.Add(leftHandLandmarks.Landmark[landmark_index].X);
+                left_hand_y.Add(leftHandLandmarks.Landmark[landmark_index].Y);
+            }
+            if (rightHandLandmarks == null)
+            {
+                right_hand_x.Add(0);
+                right_hand_y.Add(0);
+            }
+            else
+            {
+                right_hand_x.Add(rightHandLandmarks.Landmark[landmark_index].X);
+                right_hand_y.Add(rightHandLandmarks.Landmark[landmark_index].Y);
+            }
+        }
+
+        // TODO: Add normalization
+        (left_hand_x, left_hand_y) = normalizeHand(left_hand_x, left_hand_y);
+        (right_hand_x, right_hand_y) = normalizeHand(right_hand_x, right_hand_y);
+
+
+        if (keypoints_buffer.Count >= 10)
+        {
+            keypoints_buffer.RemoveAt(0);
+        }
+
+        List<float> keypoints = new List<float>();
+        for (int i = 0; i < pose_x.Count; i++)
+        {
+            keypoints.Add(pose_x[i]);
+            keypoints.Add(pose_y[i]);
+        }
+        for (int i = 0; i < left_hand_x.Count; i++)
+        {
+            keypoints.Add(left_hand_x[i]);
+            keypoints.Add(left_hand_y[i]);
+        }
+        for (int i = 0; i < right_hand_x.Count; i++)
+        {
+            keypoints.Add(right_hand_x[i]);
+            keypoints.Add(right_hand_y[i]);
+        }
+
+        keypoints_buffer.Add(keypoints);
+    }
+
+    public List<List<float>> getAllKeypoints()
+    {
+        if (keypoints_buffer.Count < 10)
+        {
+            return null;
+        }
+        return keypoints_buffer;
+    }
+}

Assets/MediaPipeUnity/Common/Scripts/KeypointManager.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 40ff941e1b34847bdb160c6950f35aec
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Common/Scripts/MediaPipeUnityScripts.asmdef

@@ -0,0 +1,16 @@
+{
+    "name": "MediaPipeUnityScripts",
+    "rootNamespace": "",
+    "references": [
+        "GUID:04c4d86a70aa56c55a78c61f1ab1a56d"
+    ],
+    "includePlatforms": [],
+    "excludePlatforms": [],
+    "allowUnsafeCode": false,
+    "overrideReferences": false,
+    "precompiledReferences": [],
+    "autoReferenced": true,
+    "defineConstraints": [],
+    "versionDefines": [],
+    "noEngineReferences": false
+}

Assets/MediaPipeUnity/Common/Scripts/MediaPipeUnityScripts.asmdef.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: edc93f477bb73a743a97d6882ed330b3
+AssemblyDefinitionImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: