WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Sprint 4

Browse Source
This commit is contained in:
Jelle De Geest
2023-04-26 19:04:34 +02:00
parent 172938cec8
commit 47f8b96122
1004 changed files with 60756 additions and 117444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Assets/MediaPipeUnity/Common/Scripts/KeypointManager.cs
View File

@@ -74,12 +74,15 @@ public class KeypointManager
}
if (width > height){
delta_x = ((float)0.1)*width;
delta_y = delta_x + ((width - height)/2);
}else{
delta_y = ((float)0.1)*height;
delta_x = delta_y + ((height - width)/2);
if (width > height)
{
delta_x = ((float)0.1) * width;
delta_y = delta_x + ((width - height) / 2);
}
else
{
delta_y = ((float)0.1) * height;
delta_x = delta_y + ((height - width) / 2);
}
float starting_x = min_x - delta_x;
@@ -124,10 +127,10 @@ public class KeypointManager
float eye_left_x = pose_x[1];
float eye_left_y = pose_y[1];
float starting_x = shoulder_center_x - (bbox_size/2) * shoulder_distance;
float starting_y = eye_left_y - shoulder_distance/2;
float starting_x = shoulder_center_x - (bbox_size / 2) * shoulder_distance;
float starting_y = eye_left_y - shoulder_distance / 2;
float ending_x = shoulder_center_x + (bbox_size/2) * shoulder_distance;
float ending_x = shoulder_center_x + (bbox_size / 2) * shoulder_distance;
float ending_y = starting_y + (bbox_size - ((float)0.5)) * shoulder_distance;
float bbox_center_x = (starting_x + ending_x) / 2;

263
Assets/MediaPipeUnity/Common/Scripts/KeypointManagerEmbedding.cs Normal file
View File

@@ -0,0 +1,263 @@
using Mediapipe;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
public class KeypointManagerEmbedding
{
private int leftShoulderIndex = 11;
private int rightShoulderIndex = 12;
private int neckIndex = 33;
private int noseIndex = 0;
private int leftEyeIndex = 2;
private List<int> pose_indices = new List<int> { 0, 33, 5, 2, 8, 7, 12, 11, 14, 13, 16, 15 };
private List<int> hand_indices = new List<int> { 0, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17, 4, 3, 2, 1 };
private static int BUFFER_SIZE = 10;
private List<List<List<float>>> keypointsBuffer;
public KeypointManagerEmbedding()
{
keypointsBuffer = new List<List<List<float>>>();
}
private (List<float>, List<float>) NormalizeHand(List<float> handX, List<float> handY)
{
var xValues = new List<float>();
var yValues = new List<float>();
for (int i = 0; i < handX.Count; i++)
{
if (handX[i] != 0)
{
xValues.Add(handX[i]);
}
if (handY[i] != 0)
{
yValues.Add(handY[i]);
}
}
if (xValues.Count == 0 || yValues.Count == 0)
{
return (handX, handY);
}
float width = xValues.Max() - xValues.Min();
float height = yValues.Max() - yValues.Min();
float delta_x, delta_y;
if (width > height)
{
delta_x = 0.1f * width;
delta_y = delta_x + ((width - height) / 2f);
}
else
{
delta_y = 0.1f * height;
delta_x = delta_y + ((height - width) / 2f);
}
var startingPoint = new Vector2(xValues.Min() - delta_x, yValues.Min() - delta_y);
var endingPoint = new Vector2(xValues.Max() + delta_x, yValues.Max() + delta_y);
if (endingPoint.x - startingPoint.x == 0f || endingPoint.y - startingPoint.y == 0f)
{
return (handX, handY);
}
// normalize keypoints
for (int i = 0; i < handX.Count; i++)
{
handX[i] = (handX[i] - startingPoint.x) / (endingPoint.x - startingPoint.x);
handY[i] = (handY[i] - startingPoint.y) / (endingPoint.y - startingPoint.y);
}
return (handX, handY);
}
private (List<float>, List<float>) NormalizePose(List<float> poseX, List<float> poseY)
{
var leftShoulder = new Vector2(poseX[leftShoulderIndex], poseY[leftShoulderIndex]);
var rightShoulder = new Vector2(poseX[rightShoulderIndex], poseY[rightShoulderIndex]);
var neck = new Vector2(poseX[neckIndex], poseY[neckIndex]);
var nose = new Vector2(poseX[noseIndex], poseY[noseIndex]);
// Prevent from even starting the analysis if some necessary elements are not present
if ((leftShoulder.x == 0 || rightShoulder.x == 0 ||
(leftShoulder.x == rightShoulder.x && leftShoulder.y == rightShoulder.y)) &&
(neck.x == 0 || nose.x == 0 || (neck.x == nose.x && neck.y == nose.y)))
{
return (poseX, poseY);
}
float shoulderDistance, headMetric;
if (leftShoulder.x != 0 && rightShoulder.x != 0 &&
(leftShoulder.x != rightShoulder.x || leftShoulder.y != rightShoulder.y))
{
shoulderDistance = Mathf.Sqrt(Mathf.Pow(leftShoulder.x - rightShoulder.x, 2) + Mathf.Pow(leftShoulder.y - rightShoulder.y, 2));
headMetric = shoulderDistance;
}
else
{
float neckNoseDistance = Mathf.Sqrt(Mathf.Pow(neck.x - nose.x, 2) + Mathf.Pow(neck.y - nose.y, 2));
headMetric = neckNoseDistance;
}
// Set the starting and ending point of the normalization bounding box
var startingPoint = new Vector2(poseX[neckIndex] - 3 * headMetric, poseY[leftEyeIndex] + headMetric);
var endingPoint = new Vector2(poseX[neckIndex] + 3 * headMetric, startingPoint.y - 6 * headMetric);
if (startingPoint.x < 0)
{
startingPoint.x = 0;
}
if (startingPoint.y < 0)
{
startingPoint.y = 0;
}
if (endingPoint.x < 0)
{
endingPoint.x = 0;
}
if (endingPoint.y < 0)
{
endingPoint.y = 0;
}
// Normalize the keypoints
for (int i = 0; i < poseX.Count; i++)
{
poseX[i] = (poseX[i] - startingPoint.x) / (endingPoint.x - startingPoint.x);
poseY[i] = (poseY[i] - endingPoint.y) / (startingPoint.y - endingPoint.y);
}
return (poseX, poseY);
}
private (List<float>, List<float>) CalculateNeck(List<float> keypointsX, List<float> keypointsY)
{
var leftShoulder = new Vector2(keypointsX[11], keypointsY[11]);
var rightShoulder = new Vector2(keypointsX[12], keypointsY[12]);
var neck = new Vector2((leftShoulder.x + rightShoulder.x) / 2, (leftShoulder.y + rightShoulder.y) / 2);
// add neck to keypoints
keypointsX.Add(neck.x);
keypointsY.Add(neck.y);
return (keypointsX, keypointsY);
}
public void AddLandmarks(NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks)
{
List<float> pose_x = new List<float>();
List<float> pose_y = new List<float>();
List<float> left_hand_x = new List<float>();
List<float> left_hand_y = new List<float>();
List<float> right_hand_x = new List<float>();
List<float> right_hand_y = new List<float>();
if (poseLandmarks == null || (leftHandLandmarks == null && rightHandLandmarks == null))
{
return;
}
if (poseLandmarks != null)
{
foreach (NormalizedLandmark landmark in poseLandmarks.Landmark)
{
pose_x.Add(landmark.X);
pose_y.Add(landmark.Y);
}
}else{
for (int i = 0; i < 33; i++)
{
pose_x.Add(0);
pose_y.Add(0);
}
}
// Add neck to pose
(pose_x, pose_y) = CalculateNeck(pose_x, pose_y);
// normalize pose
(pose_x, pose_y) = NormalizePose(pose_x, pose_y);
// now filter the pose keypoints based on the pose indeces
List<List<float>> filtered_pose = new List<List<float>>();
foreach (int index in pose_indices)
{
filtered_pose.Add(new List<float> { pose_x[index] - 0.5f, pose_y[index] - 0.5f });
}
// add hand landmarks
if (leftHandLandmarks != null)
{
foreach (NormalizedLandmark landmark in leftHandLandmarks.Landmark)
{
left_hand_x.Add(landmark.X);
left_hand_y.Add(landmark.Y);
}
}else{
for (int i = 0; i < 21; i++)
{
left_hand_x.Add(0);
left_hand_y.Add(0);
}
}
if (rightHandLandmarks != null)
{
foreach (NormalizedLandmark landmark in rightHandLandmarks.Landmark)
{
right_hand_x.Add(landmark.X);
right_hand_y.Add(landmark.Y);
}
}else{
for (int i = 0; i < 21; i++)
{
right_hand_x.Add(0);
right_hand_y.Add(0);
}
}
// normalize the hands
(left_hand_x, left_hand_y) = NormalizeHand(left_hand_x, left_hand_y);
(right_hand_x, right_hand_y) = NormalizeHand(right_hand_x, right_hand_y);
// now filter the hand keypoints based on the hand indeces
List<List<float>> filtered_left_hand = new List<List<float>>();
List<List<float>> filtered_right_hand = new List<List<float>>();
foreach (int index in hand_indices)
{
filtered_left_hand.Add(new List<float> { left_hand_x[index] - 0.5f, left_hand_y[index] - 0.5f });
filtered_right_hand.Add(new List<float> { right_hand_x[index] - 0.5f, right_hand_y[index] - 0.5f });
}
// add the filtered keypoints together in one list
List<List<float>> filtered_keypoints = new List<List<float>>();
filtered_keypoints.AddRange(filtered_pose);
filtered_keypoints.AddRange(filtered_left_hand);
filtered_keypoints.AddRange(filtered_right_hand);
keypointsBuffer.Add(filtered_keypoints);
if (keypointsBuffer.Count > BUFFER_SIZE)
{
keypointsBuffer.RemoveAt(0);
}
}
public List<List<List<float>>> GetKeypoints()
{
if (keypointsBuffer.Count < BUFFER_SIZE){
return null;
}
return keypointsBuffer;
}
}

2
Assets/MediaPipeUnity/Scripts/Feedback.cs.meta → Assets/MediaPipeUnity/Common/Scripts/KeypointManagerEmbedding.cs.meta
View File

@@ -1,5 +1,5 @@
fileFormatVersion: 2
guid: 44e682a32ee15cc489bf50f3a06f717b
guid: 8978a7c17464d4fa8ab9f33be45a2bb6
MonoImporter:
externalObjects: {}
serializedVersion: 2

8
Assets/MediaPipeUnity/Interfaces.meta Normal file
View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 73c615986873dc246893879daf74c05d
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

14
Assets/MediaPipeUnity/Interfaces/Listener.cs Normal file
View File

@@ -0,0 +1,14 @@
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
/// <summary>
/// Listener interface with an IEnumerator as its processing-function.
/// </summary>
public interface Listener
{
/// <summary>
/// The function that is called by the publisher.
/// </summary>
/// <returns></returns>
public IEnumerator ProcessIncomingCall();
}

11
Assets/MediaPipeUnity/Interfaces/Listener.cs.meta Normal file
View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: e4c1da9896d9ba2449549a016b5fd15e
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

14
Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef Normal file
View File

@@ -0,0 +1,14 @@
{
"name": "SignPredictorInterfaces",
"rootNamespace": "",
"references": [],
"includePlatforms": [],
"excludePlatforms": [],
"allowUnsafeCode": false,
"overrideReferences": false,
"precompiledReferences": [],
"autoReferenced": true,
"defineConstraints": [],
"versionDefines": [],
"noEngineReferences": false
}

7
Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta Normal file
View File

@@ -0,0 +1,7 @@
fileFormatVersion: 2
guid: f55a02e98b01bc849b30d9650ccd8f15
AssemblyDefinitionImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

41
Assets/MediaPipeUnity/Prefabs/Feedback.prefab
View File

@@ -9,7 +9,6 @@ GameObject:
serializedVersion: 6
m_Component:
- component: {fileID: 4318122119930585316}
- component: {fileID: 4318122119930585317}
m_Layer: 5
m_Name: Feedback
m_TagString: Untagged
@@ -39,22 +38,6 @@ RectTransform:
m_AnchoredPosition: {x: 0, y: 200}
m_SizeDelta: {x: 500, y: 150}
m_Pivot: {x: 0.5, y: 0}
--- !u!114 &4318122119930585317
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122119930585319}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Name:
m_EditorClassIdentifier:
feedbackText: {fileID: 4318122120222767928}
feedbackProgress: {fileID: 4318122119968934242}
feedbackProgressImage: {fileID: 4318122120334233319}
signPredictor: {fileID: 0}
--- !u!1 &4318122119968934244
GameObject:
m_ObjectHideFlags: 0
@@ -113,7 +96,7 @@ MonoBehaviour:
m_SelectOnDown: {fileID: 0}
m_SelectOnLeft: {fileID: 0}
m_SelectOnRight: {fileID: 0}
m_Transition: 1
m_Transition: 0
m_Colors:
m_NormalColor: {r: 1, g: 1, b: 1, a: 1}
m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1}
@@ -133,7 +116,7 @@ MonoBehaviour:
m_PressedTrigger: Pressed
m_SelectedTrigger: Selected
m_DisabledTrigger: Disabled
m_Interactable: 1
m_Interactable: 0
m_TargetGraphic: {fileID: 0}
m_FillRect: {fileID: 4318122120334233317}
m_HandleRect: {fileID: 0}
@@ -213,15 +196,15 @@ MonoBehaviour:
m_Calls: []
m_text: Detecteren ...
m_isRightToLeft: 0
m_fontAsset: {fileID: 11400000, guid: 8f586378b4e144a9851e7b34d9b748ee, type: 2}
m_sharedMaterial: {fileID: 2180264, guid: 8f586378b4e144a9851e7b34d9b748ee, type: 2}
m_fontAsset: {fileID: 11400000, guid: 1baf2eae62f542f4585aaf3c9c3e229a, type: 2}
m_sharedMaterial: {fileID: -2577534979213189211, guid: 1baf2eae62f542f4585aaf3c9c3e229a, type: 2}
m_fontSharedMaterials: []
m_fontMaterial: {fileID: 0}
m_fontMaterials: []
m_fontColor32:
serializedVersion: 2
rgba: 4282188031
m_fontColor: {r: 1, g: 0, b: 0.23945475, a: 1}
rgba: 4282206709
m_fontColor: {r: 0.9607843, g: 0.28627452, b: 0.23921569, a: 1}
m_enableVertexGradient: 0
m_colorMode: 3
m_fontColorGradient:
@@ -244,8 +227,8 @@ MonoBehaviour:
m_enableAutoSizing: 0
m_fontSizeMin: 18
m_fontSizeMax: 72
m_fontStyle: 1
m_HorizontalAlignment: 2
m_fontStyle: 0
m_HorizontalAlignment: 1
m_VerticalAlignment: 512
m_textAlignment: 65535
m_characterSpacing: 0
@@ -339,14 +322,14 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 0, b: 0, a: 1}
m_Color: {r: 0.9607843, g: 0.28627452, b: 0.23921569, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_Sprite: {fileID: 10905, guid: 0000000000000000f000000000000000, type: 0}
m_Sprite: {fileID: 0}
m_Type: 1
m_PreserveAspect: 0
m_FillCenter: 1
@@ -415,14 +398,14 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 1, b: 1, a: 1}
m_Color: {r: 0.5803922, g: 0.58431375, b: 0.6, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_Sprite: {fileID: 10907, guid: 0000000000000000f000000000000000, type: 0}
m_Sprite: {fileID: 0}
m_Type: 1
m_PreserveAspect: 0
m_FillCenter: 1

8
Assets/MediaPipeUnity/ScriptableObjects.meta Normal file
View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 438a3ced42dd6fc4ab38e3a16c1e43a7
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

25
Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset Normal file
View File

@@ -0,0 +1,25 @@
%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!114 &11400000
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 78a3f61c93a08c04496c49ffd10b9021, type: 3}
m_Name: ModelList
m_EditorClassIdentifier:
models:
- index: 0
modelWINDOWS: {fileID: 0}
modelMAC: {fileID: 0}
- index: 1
modelWINDOWS: {fileID: 8538825877217656561, guid: fdbf401e965a6bf4a87637cd519f2715, type: 3}
modelMAC: {fileID: 8538825877217656561, guid: be31548ec7e7544fe9828b14657bb40b, type: 3}
- index: 2
modelWINDOWS: {fileID: 8538825877217656561, guid: fa63c40c78ba548468cad97b15cdc6c9, type: 3}
modelMAC: {fileID: 8538825877217656561, guid: 17fb70e1c284e44da8083b36bb6afcb8, type: 3}
currentModelIndex: 2

8
Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset.meta Normal file
View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 39516e4e6e56f0f4f80647d9c4d8034c
NativeFormatImporter:
externalObjects: {}
mainObjectFileID: 11400000
userData:
assetBundleName:
assetBundleVariant:

36
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs Normal file
View File

@@ -0,0 +1,36 @@
using System.Collections;
using UnityEngine;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public abstract class AbstractFeedback : MonoBehaviour, Listener
{
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// The function that is called by the publisher on all its listeners
/// </summary>
/// <returns></returns>
public IEnumerator ProcessIncomingCall()
{
yield return StartCoroutine(UpdateFeedback());
}
/// <summary>
/// A function to add yourself as listener to the signPredictor you are holding
/// </summary>
public void AddSelfAsListener()
{
signPredictor.listeners.Add(this);
}
/// <summary>
/// The function that holds the logic to process the new probabilities of the signPredictor
/// </summary>
/// <returns></returns>
protected abstract IEnumerator UpdateFeedback();
}

11
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta Normal file
View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 7b5ac794337a54143a6e3077483d96c9
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

182
Assets/MediaPipeUnity/Scripts/Feedback.cs
View File

@@ -1,182 +0,0 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public class Feedback : MonoBehaviour
{
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// Callback for getting the correct sign
/// </summary>
public Func<string> getSignCallback;
/// <summary>
/// Callback to initiate the next sign
/// </summary>
public UnityAction<string> predictSignCallback;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
/// </summary>
private DateTime timer;
/// <summary>
/// Current predicted sign
/// </summary>
private string predictedSign = null;
/// <summary>
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
/// </summary>
private string previousIncorrectSign = null;
/// <summary>
/// Start is called before the first frame update
/// </summary>
void Start()
{
// Start the coroutine to update the scale every 200 milliseconds
StartCoroutine(UpdateFeedback());
}
/// <summary>
/// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
/// If a high enough accuracy is detected, it will go to the next sign
/// </summary>
/// <returns></returns>
IEnumerator UpdateFeedback()
{
while (true)
{
if (getSignCallback != null && predictSignCallback != null)
{
// Get current sign
string currentSign = getSignCallback();
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null){
if (accuracy > 0.98)
{
// TODO: fix emojis
feedbackText.text = "✨ Perfect ✨";
Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.95)
{
feedbackText.text = "Super!";
Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if(feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
// Wait for 200 milliseconds before updating the scale again
yield return new WaitForSeconds(0.2f);
}
}
}

6
Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef
View File

@@ -3,10 +3,12 @@
"rootNamespace": "",
"references": [
"GUID:6055be8ebefd69e48b49212b09b47b2f",
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
"GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
"GUID:edc93f477bb73a743a97d6882ed330b3",
"GUID:58e104b97fb3752438ada2902a36dcbf"
"GUID:58e104b97fb3752438ada2902a36dcbf",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:f55a02e98b01bc849b30d9650ccd8f15",
"GUID:d23f64cfd3b314bb4a18a8284c99bf5e"
],
"includePlatforms": [],
"excludePlatforms": [],

982
Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

File diff suppressed because it is too large Load Diff