deleted stale dependency

Removed items in abstract class that don't need to be there

.gitlab/merge_request_templates/Default.md

@@ -1,41 +0,0 @@
-## Description
-
-_Please provide a brief summary of the changes in this merge request._
-
-_If possible, add a short screengrab or some screenshots of the changes._
-
-## Testing Instructions
-
-_Please provide instructions on how the code reviewers can test your changes:_
-
-1. [Step 1]
-2. [Step 2]
-3. [Step 3]
-4. ...
-
-_Please include any specific information on test data, configurations, or other requirements that are necessary to properly test the changes._
-
-Once you've tested the changes, please confirm that they work as expected and that there are no regressions or unexpected side effects. If any issues are discovered during testing, please include detailed steps to reproduce the issue in the merge request comments. Thank you!
-
-## Related Issues
-
-_Please list any related issues or pull requests that are relevant to this merge request._
-_E.g. WES-XXX-..._
-
-## Known bugs or issues
-
-_Please list any known bugs or issues related to the changes in this merge request._
-
-## Checklist
-
-- [ ] I have filled in this template.
-- [ ] I have tested my changes thoroughly (both in the editor + **build and run (ctrl+B)**!).
-- [ ] I have added appropriate unit tests.
-- [ ] I have updated the user documentation as necessary.
-- [ ] Code reviewed by 2 people.
-
-## Additional Notes
-
-_Please add any additional notes or comments that may be helpful for reviewers to understand your changes._
-
-/assign_reviewer @wesign/unityappreviewers 

Assets/Common/Interfaces/ModelIndex.cs

@@ -0,0 +1,11 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+/// <summary>
+/// This enum is used to identify each of the SignLanguage models
+/// </summary>
+public enum ModelIndex 
+{
+    FINGERSPELLING,
+    NONE
+}

Assets/Common/Interfaces/ModelIndex.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 44e682a32ee15cc489bf50f3a06f717b
+guid: 6dbd5e1100bc81648b52206df369d0a1
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Assets/Common/Interfaces/ModelList.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.Barracuda;
+/// <summary>
+/// This scriptable will hold tupples of Courseindices and models
+/// </summary>
+[CreateAssetMenu(menuName = "Create new Scriptable/ModelList")]
+public class ModelList : ScriptableObject
+{
+    /// <summary>
+    /// Small class to link a model to a courseIndex irrespective of its position in a list
+    /// </summary>
+    [Serializable]
+    public class ModelTuple
+    {
+        /// <summary>
+        /// ModelIndex to which the model corresponds
+        /// </summary>
+        public ModelIndex index;
+        /// <summary>
+        /// The model itself
+        /// </summary>
+        public NNModel model;
+    }
+
+
+    /// <summary>
+    /// Index of the currently active model
+    /// </summary>
+    public int currentModelIndex = 0;
+
+    /// <summary>
+    /// A list of all the models
+    /// </summary>
+    public List<ModelTuple> models = new List<ModelTuple>();
+
+    /// <summary>
+    /// Get a model by modelindex
+    /// </summary>
+    /// <param name="modelIndex">ModelIndex of the model</param>
+    /// <returns>Model associated with this index, null if no model was found</returns>
+    public NNModel GetCurrentModel()
+    {
+        return models.Find(x => x.model == models[currentModelIndex].model)?.model;
+    }
+
+    /// <summary>
+    /// Function to find a model-index in the list based on its index
+    /// </summary>
+    /// <param name="title"></param>
+    public void SetCurrentModel(ModelIndex index)
+    {
+        currentModelIndex = models.FindIndex((m) => m.index == index);
+    }
+}

Assets/Common/Interfaces/ModelList.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 78a3f61c93a08c04496c49ffd10b9021
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Common/Interfaces/Theme.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using Unity.Barracuda;
 using UnityEngine;
 
 /// <summary>
@@ -24,9 +23,9 @@ public class Theme : ScriptableObject
     public ThemeIndex index;
 
     /// <summary>
-    /// Reference to the model used in the SignPredictor
+    /// The index of the model you want to use
     /// </summary>
-    public NNModel model;
+    public ModelIndex modelIndex;
 
     /// <summary>
     /// List of all learnable words/letters

Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef

@@ -8,7 +8,8 @@
         "CommonScripts",
         "InterfacesScripts",
         "Unity.TextMeshPro",
-        "AccountsScripts"
+        "AccountsScripts",
+        "SignPredictor"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Common/ScriptableObjects/AnimalsTheme.asset

@@ -16,6 +16,7 @@ MonoBehaviour:
   description: Van vis tot leeuw
   index: 2
   model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Walvis
     image: {fileID: 21300000, guid: 2b01165a5836ab14593d7a5862bd6793, type: 3}

Assets/Common/ScriptableObjects/ClothesTheme.asset

@@ -15,6 +15,8 @@ MonoBehaviour:
   title: Kleren en Kleuren
   description: Van rok tot sok
   index: 1
+  model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Blauw
     image: {fileID: 21300000, guid: 182fb89eba9c64041bef31ca35c4bcd8, type: 3}

Assets/Common/ScriptableObjects/FamilyTheme.asset

@@ -15,6 +15,8 @@ MonoBehaviour:
   title: Familie
   description: Van generatie tot generatie
   index: 6
+  model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Broer
     image: {fileID: 21300000, guid: eecf67266f150f1489717049489cf16d, type: 3}

Assets/Common/ScriptableObjects/FoodTheme.asset

@@ -16,6 +16,7 @@ MonoBehaviour:
   description: Van kers tot pompoen
   index: 3
   model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Aardappel
     image: {fileID: 21300000, guid: 2610cdbc24a125f43ada7fed67d8f51b, type: 3}

Assets/Common/ScriptableObjects/HobbyTheme.asset

@@ -15,6 +15,8 @@ MonoBehaviour:
   title: Hobbies
   description: Van schilderen tot reizen
   index: 4
+  model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Dansen
     image: {fileID: 21300000, guid: 6d405f607ae817744b49f921f0611088, type: 3}

Assets/Common/ScriptableObjects/HouseTheme.asset

@@ -15,6 +15,8 @@ MonoBehaviour:
   title: Huis beschrijven
   description: Van zetel tot villa
   index: 5
+  model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Keuken
     image: {fileID: 21300000, guid: b17ce5bf59092b847b084d3400e7a1b4, type: 3}

Assets/Common/Scripts/CommonScripts.asmdef

@@ -5,8 +5,8 @@
         "GUID:6055be8ebefd69e48b49212b09b47b2f",
         "GUID:63c63e721f65ebb7d871cb9ef49f4752",
         "GUID:1631ed2680c61245b8211d943c1639a8",
-        "GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
-        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
+        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
+        "GUID:d0b6b39a21908f94fbbd9f2c196a9725"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Common/Scripts/CourseActivityScreen.cs

@@ -56,8 +56,8 @@ public class CourseActivityScreen : MonoBehaviour
         Course course = courseList.courses[index];
 
         // vvv TEMPORARY STUFF vvv
-        playButton.SetActive(course.theme.model != null);
-        previewButton.SetActive(course.theme.model == null);
+        playButton.SetActive(course.theme.modelIndex != ModelIndex.NONE);
+        previewButton.SetActive(course.theme.modelIndex == ModelIndex.NONE);
         // ^^^ TEMPORARY STUFF ^^^
 
         title.text = course.title;

Assets/Common/Tests/CommonTests.asmdef

@@ -5,7 +5,9 @@
         "UnityEngine.TestRunner",
         "UnityEditor.TestRunner",
         "CommonScripts",
-        "InterfacesScripts"
+        "InterfacesScripts",
+        "Unity.Barracuda",
+        "SignPredictor"
     ],
     "includePlatforms": [
         "Editor"

Assets/Common/Tests/ModelListTest.cs

@@ -0,0 +1,80 @@
+using NUnit.Framework;
+using Unity.Barracuda;
+using UnityEngine;
+/// <summary>
+/// Test the ModelList class
+/// </summary>
+[TestFixture]
+public class ModelListTest
+{
+    private ModelList modelList;
+
+    /// <summary>
+    /// Setup a ModelList with all possible Models in the enum
+    /// </summary>
+    [SetUp]
+    public void Setup_Model()
+    {
+        modelList = ScriptableObject.CreateInstance<ModelList>();
+
+        // Add a Model for each index in the enum
+
+        // Dumb way to access each index in the enum, couldn't find a different way to do it though
+        foreach (var field in typeof(ModelIndex).GetFields())
+        {
+            if (field.IsLiteral)
+            {
+                ModelIndex value = (ModelIndex)field.GetValue(null);
+                string name = field.Name;
+                ModelList.ModelTuple model = new ModelList.ModelTuple();
+                // This is all we will need to distinguish 
+                model.index = value;
+
+                // Insert in front to guarantee that ModelIndex will not line up with listIndex
+                modelList.models.Insert(0, model);
+            }
+        }
+    }
+    /// <summary>
+    /// Check if current model can be correctly gotten as current via GetCurrentModel
+    /// </summary>
+    [Test]
+    public void TestGetCurrentModel()
+    {
+        System.Random random = new System.Random();
+        ModelIndex value = (ModelIndex)random.Next(modelList.models.Count);
+        modelList.SetCurrentModel(value);
+
+        Assert.AreEqual(modelList.models[modelList.currentModelIndex].model, modelList.GetCurrentModel());
+
+        // Check if empty model fails gracefully (returns null)
+        Assert.IsNull(ScriptableObject.CreateInstance<ModelList>().GetCurrentModel());
+    }
+
+    /// <summary>
+    /// Check if all models can be correctly set as current via SetCurrentModel
+    /// </summary>
+    [Test]
+    public void TestSetCurrentModel()
+    {
+        foreach (var field in typeof(ModelIndex).GetFields())
+        {
+            if (field.IsLiteral)
+            {
+                ModelIndex value = (ModelIndex)field.GetValue(null);
+                string name = field.Name;
+                modelList.SetCurrentModel(value);
+
+                // Fetch the current model and check if its name is the same as the one we made into the current one 
+                ModelList.ModelTuple m = modelList.models[modelList.currentModelIndex];
+
+                Assert.AreEqual(m.index, value);
+                Assert.IsTrue(m.model is NNModel || m.model is null);
+            }
+        }
+        ModelList emptyList = ScriptableObject.CreateInstance<ModelList>();
+        emptyList.SetCurrentModel(ModelIndex.FINGERSPELLING);
+
+        Assert.IsTrue(emptyList.currentModelIndex == -1);
+    }
+}

Assets/Common/Tests/ModelListTest.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 1313e0cc80244354eb6e2d0c1e891941
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Courses/Scenes/TemplateCourse.unity

@@ -490,6 +490,10 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  feedbackText: {fileID: 4318122121437849762}
+  feedbackProgress: {fileID: 4318122121437849761}
+  feedbackProgressImage: {fileID: 4318122121437849760}
+  signPredictor: {fileID: 883853268}
   previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   feedbackProgressBar: {fileID: 4318122121437849759}
   previewMessage: {fileID: 2070775951}
@@ -505,7 +509,6 @@ MonoBehaviour:
   ResultsDecription: {fileID: 100123246}
   CoursesButton: {fileID: 839294691}
   timeSpent: {fileID: 77614869}
-  feedback: {fileID: 1714882683}
 --- !u!1 &361280475
 GameObject:
   m_ObjectHideFlags: 0
@@ -1222,8 +1225,8 @@ MonoBehaviour:
   m_OnClick:
     m_PersistentCalls:
       m_Calls:
-      - m_Target: {fileID: 301088551}
-        m_TargetAssemblyTypeName: TemplateCourse, Assembly-CSharp
+      - m_Target: {fileID: 1335886461}
+        m_TargetAssemblyTypeName: BackButton, CommonScripts
         m_MethodName: Back
         m_Mode: 1
         m_Arguments:
@@ -1301,7 +1304,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+  modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 378145456}
@@ -1839,6 +1842,17 @@ RectTransform:
   m_CorrespondingSourceObject: {fileID: 8299246693487308515, guid: 3bccdf365a4fbea4d8fa1aa461d3dc5c, type: 3}
   m_PrefabInstance: {fileID: 1335886459}
   m_PrefabAsset: {fileID: 0}
+--- !u!114 &1335886461 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4518652150503380115, guid: 3bccdf365a4fbea4d8fa1aa461d3dc5c, type: 3}
+  m_PrefabInstance: {fileID: 1335886459}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: c3dd279b546423e4a8a1b28819a6c4a1, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
 --- !u!1 &1383144366
 GameObject:
   m_ObjectHideFlags: 0
@@ -2252,17 +2266,6 @@ RectTransform:
   m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 4318122121437849758}
   m_PrefabAsset: {fileID: 0}
---- !u!114 &1714882683 stripped
-MonoBehaviour:
-  m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
-  m_PrefabInstance: {fileID: 4318122121437849758}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 4318122121437849759}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
 --- !u!1 &1773033262
 GameObject:
   m_ObjectHideFlags: 0
@@ -2838,6 +2841,10 @@ PrefabInstance:
       propertyPath: m_LocalEulerAnglesHint.z
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+      propertyPath: m_Enabled
+      value: 0
+      objectReference: {fileID: 0}
     - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
       propertyPath: signPredictor
       value: 
@@ -2865,3 +2872,36 @@ GameObject:
   m_CorrespondingSourceObject: {fileID: 4318122119930585319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 4318122121437849758}
   m_PrefabAsset: {fileID: 0}
+--- !u!114 &4318122121437849760 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &4318122121437849761 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &4318122121437849762 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 

Assets/Courses/Scripts/CourseScripts.asmdef

@@ -6,7 +6,9 @@
         "AccountsScripts",
         "InterfacesScripts",
         "SignPredictor",
-        "Unity.Barracuda"
+        "Unity.Barracuda",
+        "Tween",
+        "SignPredictorInterfaces"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Courses/Scripts/TemplateCourse.cs

@@ -1,14 +1,16 @@
 using System;
+using System.Collections;
 using TMPro;
 using Unity.Barracuda;
 using UnityEngine;
 using UnityEngine.UI;
 using UnityEngine.Video;
+using DigitalRuby.Tween;
 
 /// <summary>
 /// TemplateCourse scene manager
 /// </summary>
-public class TemplateCourse : MonoBehaviour
+public class TemplateCourse : AbstractFeedback
 {
     // vvv TEMPORARY STUFF vvv
     public NNModel previewModel;
@@ -112,10 +114,37 @@ public class TemplateCourse : MonoBehaviour
     /// </summary>
     public TMP_Text timeSpent;
 
+
     /// <summary>
-    /// Reference to the feedback script on the Feedback prefab
+    /// Reference to the feedback field
     /// </summary>
-    public Feedback feedback;
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
 
     /// <summary>
     /// This function is called when the script is initialised.
@@ -124,17 +153,23 @@ public class TemplateCourse : MonoBehaviour
     /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
     /// Then it sets up the course-screen to display relevant information from the course-scriptable.
     /// </summary>
-    void Awake()
+    void Start()
+    {
+        StartGameController();
+
+        signPredictor.SetModel(course.theme.modelIndex);
+        AddSelfAsListener();
+    }
+
+    public void StartGameController()
     {
         // Setting up course
         course = courselist.courses[courselist.currentCourseIndex];
-        feedback.signPredictor.model = course.theme.model;
         maxWords = course.theme.learnables.Count;
 
         // vvv TEMPORARY STUFF vvv
-        feedbackProgressBar.SetActive(course.theme.model != null);
-        previewMessage.SetActive(course.theme.model == null);
-        feedback.signPredictor.model = previewModel;
+        feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE);
+        previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE);
         // ^^^ TEMPORARY STUFF ^^^
 
         // Create entry in current user for keeping track of progress
@@ -163,23 +198,6 @@ public class TemplateCourse : MonoBehaviour
         ResultPanel.SetActive(false);
         // Set the startTime
         startMoment = DateTime.Now;
-
-        // Set callbacks
-        feedback.getSignCallback = () =>
-        {
-            if (currentWordIndex < course.theme.learnables.Count)
-            {
-                return course.theme.learnables[currentWordIndex].name;
-            }
-            return null;
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            if (sign == course.theme.learnables[currentWordIndex].name)
-            {
-                NextSign();
-            }
-        };
     }
 
     /// <summary>
@@ -286,4 +304,98 @@ public class TemplateCourse : MonoBehaviour
         progress.AddOrUpdate<float>("courseProgress", 1f);
         userList.Save();
     }
+
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = course.theme.learnables[currentWordIndex].name;
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        CheckEquality(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            CheckEquality(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+        
+    }
+
+    private void CheckEquality(string predicted)
+    {
+        if(predicted == course.theme.learnables[currentWordIndex].name)
+        {
+            NextSign();
+        }
+    }
 }

Assets/Hangman/Scenes/Hangman.unity

@@ -2570,7 +2570,7 @@ Transform:
   m_ConstrainProportionsScale: 0
   m_Children: []
   m_Father: {fileID: 0}
-  m_RootOrder: 4
+  m_RootOrder: 5
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
 --- !u!1 &896343212
 GameObject:
@@ -3507,7 +3507,7 @@ RectTransform:
   - {fileID: 1892638588}
   - {fileID: 56162990}
   m_Father: {fileID: 0}
-  m_RootOrder: 2
+  m_RootOrder: 3
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
   m_AnchorMin: {x: 0, y: 0}
   m_AnchorMax: {x: 0, y: 0}
@@ -3526,6 +3526,10 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 2db44635e0eb1e9429a2e6195785364d, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  feedbackText: {fileID: 5233312448025626833}
+  feedbackProgress: {fileID: 5233312447201393291}
+  feedbackProgressImage: {fileID: 5233312447919013134}
+  signPredictor: {fileID: 1991376311}
   themelist: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
   letterPrefab: {fileID: 4639383499500021565, guid: c3e66e8957864914cb022af914df6a28, type: 3}
   letterContainer: {fileID: 1870283439}
@@ -3550,7 +3554,6 @@ MonoBehaviour:
   Scoreboard: {fileID: 1007532375}
   EntriesGrid: {fileID: 1391137944}
   scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3}
-  feedback: {fileID: 5233312447513285388}
   gottogamebutton: {fileID: 1581633295}
 --- !u!1001 &1290865991
 PrefabInstance:
@@ -6245,7 +6248,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+  modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 1649505745}
@@ -6263,7 +6266,7 @@ Transform:
   m_ConstrainProportionsScale: 0
   m_Children: []
   m_Father: {fileID: 0}
-  m_RootOrder: 5
+  m_RootOrder: 2
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
 --- !u!1 &2001212056
 GameObject:
@@ -6566,22 +6569,6 @@ GameObject:
   m_NavMeshLayer: 0
   m_StaticEditorFlags: 0
   m_IsActive: 1
---- !u!114 &5233312447513285388
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 5233312447513285390}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  feedbackText: {fileID: 0}
-  feedbackProgress: {fileID: 0}
-  feedbackProgressImage: {fileID: 0}
-  signPredictor: {fileID: 1991376311}
 --- !u!224 &5233312447513285389
 RectTransform:
   m_ObjectHideFlags: 0
@@ -6597,7 +6584,7 @@ RectTransform:
   - {fileID: 5233312448025626847}
   - {fileID: 5233312447201393292}
   m_Father: {fileID: 0}
-  m_RootOrder: 3
+  m_RootOrder: 4
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
   m_AnchorMin: {x: 0.5, y: 0}
   m_AnchorMax: {x: 0.5, y: 0}
@@ -6613,7 +6600,6 @@ GameObject:
   serializedVersion: 6
   m_Component:
   - component: {fileID: 5233312447513285389}
-  - component: {fileID: 5233312447513285388}
   m_Layer: 5
   m_Name: Feedback
   m_TagString: Untagged

Assets/Hangman/Scripts/HangmanGameController.cs

@@ -1,11 +1,13 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using TMPro;
 using UnityEngine;
 using UnityEngine.UI;
+using DigitalRuby.Tween;
 
-public class HangmanGameController : MonoBehaviour
+public class HangmanGameController : AbstractFeedback
 {
     /// <summary>
     /// The scriptable with all the themes, will be used to select a random word for hangman.
@@ -200,11 +202,6 @@ public class HangmanGameController : MonoBehaviour
     /// </summary>
     public GameObject scoreboardEntry;
 
-    /// <summary>
-    /// Accuracy feeback object
-    /// </summary>
-    public Feedback feedback;
-
     /// <summary>
     /// The button to go into the game
     /// </summary>
@@ -215,8 +212,48 @@ public class HangmanGameController : MonoBehaviour
     /// </summary>
     private String currentsign = "";
 
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
     // Start is called before the first frame update
     void Start()
+    {
+        StartController();
+
+        signPredictor.SetModel(ModelIndex.FINGERSPELLING);
+        AddSelfAsListener();
+    }
+    /// <summary>
+    /// Called at the start of the scene AND when the scene is replayed
+    /// </summary>
+    public void StartController()
     {
         // Make sure the mode starts at zero
         mode = 0;
@@ -240,16 +277,6 @@ public class HangmanGameController : MonoBehaviour
             user.minigames.Add(progress);
         }
         userList.Save();
-
-        // Set calllbacks
-        feedback.getSignCallback = () =>
-        {
-            return "A";
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            currentsign = sign;
-        };
     }
 
     /// <summary>
@@ -278,7 +305,7 @@ public class HangmanGameController : MonoBehaviour
         DeleteWord();
         DisplayWord(currentWord);
 
-        replayButton.onClick.AddListener(Start);
+        replayButton.onClick.AddListener(StartController);
         // Call to display the first image, corresponding to a clean image.
         ChangeSprite();
     }
@@ -349,7 +376,7 @@ public class HangmanGameController : MonoBehaviour
     {
         if (mode == 1)
         {
-            if (currentsign != "")
+            if (currentsign != "" && currentsign != null)
             {
                 char letter = currentsign.ToLower()[0];
                 currentsign = "";
@@ -390,7 +417,7 @@ public class HangmanGameController : MonoBehaviour
                 // For the first input char given by the user, check if the letter is in the word that needs to be spelled.
 
                 // Check to make sure the inputfield is not empty
-                if (currentsign != "")
+                if (currentsign != null && currentsign != "")
                 {
                     char firstLetter = currentsign.ToLower()[0];
                     currentsign = "";
@@ -540,21 +567,6 @@ public class HangmanGameController : MonoBehaviour
         }
     }
 
-    /// <summary>
-    /// Randomly shuffle the list of words
-    /// </summary>
-    private void ShuffleWords()
-    {
-        for (int i = words.Length - 1; i > 0; i--)
-        {
-            // Generate a random index between 0 and i (inclusive)
-            int j = UnityEngine.Random.Range(0, i + 1);
-
-            // Swap the values at indices i and j
-            (words[j], words[i]) = (words[i], words[j]);
-        }
-    }
-
     /// <summary>
     /// Update and save the scores
     /// </summary>
@@ -728,4 +740,94 @@ public class HangmanGameController : MonoBehaviour
             rank++;
         }
     }
+
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = "A";
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        CheckEquality(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            CheckEquality(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+
+    }
+    private void CheckEquality(string sign)
+    {
+        currentsign = sign;
+    }
 }

Assets/MediaPipeUnity/Interfaces.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 73c615986873dc246893879daf74c05d
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Interfaces/Listener.cs

@@ -0,0 +1,8 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public interface Listener
+{
+    public IEnumerator ProcessIncomingCall();
+}

Assets/MediaPipeUnity/Interfaces/Listener.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: e4c1da9896d9ba2449549a016b5fd15e
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef

@@ -0,0 +1,14 @@
+{
+    "name": "SignPredictorInterfaces",
+    "rootNamespace": "",
+    "references": [],
+    "includePlatforms": [],
+    "excludePlatforms": [],
+    "allowUnsafeCode": false,
+    "overrideReferences": false,
+    "precompiledReferences": [],
+    "autoReferenced": true,
+    "defineConstraints": [],
+    "versionDefines": [],
+    "noEngineReferences": false
+}

Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: f55a02e98b01bc849b30d9650ccd8f15
+AssemblyDefinitionImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Prefabs/Feedback.prefab

@@ -9,7 +9,6 @@ GameObject:
   serializedVersion: 6
   m_Component:
   - component: {fileID: 4318122119930585316}
-  - component: {fileID: 4318122119930585317}
   m_Layer: 5
   m_Name: Feedback
   m_TagString: Untagged
@@ -39,22 +38,6 @@ RectTransform:
   m_AnchoredPosition: {x: 0, y: 200}
   m_SizeDelta: {x: 500, y: 150}
   m_Pivot: {x: 0.5, y: 0}
---- !u!114 &4318122119930585317
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 4318122119930585319}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  feedbackText: {fileID: 4318122120222767928}
-  feedbackProgress: {fileID: 4318122119968934242}
-  feedbackProgressImage: {fileID: 4318122120334233319}
-  signPredictor: {fileID: 0}
 --- !u!1 &4318122119968934244
 GameObject:
   m_ObjectHideFlags: 0

Assets/MediaPipeUnity/ScriptableObjects.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 438a3ced42dd6fc4ab38e3a16c1e43a7
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset

@@ -0,0 +1,20 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!114 &11400000
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 78a3f61c93a08c04496c49ffd10b9021, type: 3}
+  m_Name: ModelList
+  m_EditorClassIdentifier: 
+  currentModelIndex: 0
+  models:
+  - index: 0
+    model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+  - index: 1
+    model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}

Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 39516e4e6e56f0f4f80647d9c4d8034c
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 11400000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs

@@ -0,0 +1,32 @@
+using DigitalRuby.Tween;
+using Mediapipe.Unity.Tutorial;
+using System;
+using System.Collections;
+using TMPro;
+using UnityEngine;
+using UnityEngine.Events;
+using UnityEngine.UI;
+
+/// <summary>
+/// Class to display feedback during a course
+/// </summary>
+public abstract class AbstractFeedback : MonoBehaviour, Listener
+{
+
+    /// <summary>
+    /// Reference to the sign predictor
+    /// </summary>
+    public SignPredictor signPredictor;
+
+    public IEnumerator ProcessIncomingCall()
+    {
+        yield return StartCoroutine(UpdateFeedback());
+    }
+
+    public void AddSelfAsListener()
+    {
+        signPredictor.listeners.Add(this);
+    }
+
+    protected abstract IEnumerator UpdateFeedback();
+}

Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 7b5ac794337a54143a6e3077483d96c9
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Scripts/Feedback.cs

@@ -1,182 +0,0 @@
-using DigitalRuby.Tween;
-using Mediapipe.Unity.Tutorial;
-using System;
-using System.Collections;
-using TMPro;
-using UnityEngine;
-using UnityEngine.Events;
-using UnityEngine.UI;
-
-/// <summary>
-/// Class to display feedback during a course
-/// </summary>
-public class Feedback : MonoBehaviour
-{
-    /// <summary>
-    /// Reference to the feedback field
-    /// </summary>
-    public TMP_Text feedbackText;
-
-    /// <summary>
-    /// Reference to the progress bar
-    /// </summary>
-    public Slider feedbackProgress;
-
-    /// <summary>
-    /// Reference to the progress bar image, so we can add fancy colors
-    /// </summary>
-    public Image feedbackProgressImage;
-
-    /// <summary>
-    /// Reference to the sign predictor
-    /// </summary>
-    public SignPredictor signPredictor;
-
-    /// <summary>
-    /// Callback for getting the correct sign
-    /// </summary>
-    public Func<string> getSignCallback;
-
-    /// <summary>
-    /// Callback to initiate the next sign
-    /// </summary>
-    public UnityAction<string> predictSignCallback;
-
-    /// <summary>
-    /// Timer to keep track of how long a incorrect sign is performed
-    /// </summary>
-    private DateTime timer;
-
-    /// <summary>
-    /// Current predicted sign
-    /// </summary>
-    private string predictedSign = null;
-
-    /// <summary>
-    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
-    /// </summary>
-    private string previousIncorrectSign = null;
-
-    /// <summary>
-    /// Start is called before the first frame update
-    /// </summary>
-    void Start()
-    {
-        // Start the coroutine to update the scale every 200 milliseconds
-        StartCoroutine(UpdateFeedback());
-    }
-
-    /// <summary>
-    /// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
-    /// If a high enough accuracy is detected, it will go to the next sign
-    /// </summary>
-    /// <returns></returns>
-    IEnumerator UpdateFeedback()
-    {
-        while (true)
-        {
-            if (getSignCallback != null && predictSignCallback != null)
-            {
-
-                // Get current sign
-                string currentSign = getSignCallback();
-                // Get the predicted sign
-                if (signPredictor != null && signPredictor.learnableProbabilities != null &&
-                    currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
-                {    
-                    float accuracy = signPredictor.learnableProbabilities[currentSign];
-                    if (feedbackText != null && feedbackProgressImage != null){
-                        if (accuracy > 0.98)
-                        {
-                            // TODO: fix emojis
-                            feedbackText.text = "✨ Perfect ✨";
-                            Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else if (accuracy > 0.95)
-                        {
-                            feedbackText.text = "Super!";
-                            Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else if (accuracy > 0.90)
-                        {
-                            feedbackText.text = "Goed";
-                            feedbackText.color = Color.green;
-                            feedbackProgressImage.color = Color.green;
-                        }
-                        else if (accuracy > 0.80)
-                        {
-                            feedbackText.text = "Bijna...";
-                            Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else
-                        {
-                            feedbackText.text = "Detecteren...";
-                            feedbackText.color = Color.red;
-                            feedbackProgressImage.color = Color.red;
-                        }
-
-                        float oldValue = feedbackProgress.value;
-                        // use an exponential scale 
-                        float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
-                        feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
-                        {
-                            if (feedbackProgress != null)
-                            {
-                                feedbackProgress.value = t.CurrentValue;
-                            }
-                        });
-                    }
-
-                    // Check whether (in)correct sign has high accuracy
-                    foreach (var kv in signPredictor.learnableProbabilities)
-                    {
-                        if (kv.Value > 0.90)
-                        {
-                            predictedSign = kv.Key;
-                            // Correct sign
-                            if (predictedSign == currentSign)
-                            {
-                                yield return new WaitForSeconds(1.0f);
-                                predictSignCallback(predictedSign);
-                                timer = DateTime.Now;
-                                predictedSign = null;
-                                previousIncorrectSign = null;
-                            }
-                            // Incorrect sign
-                            else
-                            {
-                                if (previousIncorrectSign != predictedSign)
-                                {
-                                    timer = DateTime.Now;
-                                    previousIncorrectSign = predictedSign;
-                                }
-                                else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
-                                {
-                                    predictSignCallback(predictedSign);
-                                    timer = DateTime.Now;
-                                    predictedSign = null;
-                                    previousIncorrectSign = null;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                else if(feedbackProgress != null)
-                {
-                    
-                    feedbackProgress.value = 0.0f;
-                }
-            }
-
-            // Wait for 200 milliseconds before updating the scale again
-            yield return new WaitForSeconds(0.2f);
-        }
-    }
-}

Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef

@@ -6,7 +6,9 @@
         "GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
         "GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
         "GUID:edc93f477bb73a743a97d6882ed330b3",
-        "GUID:58e104b97fb3752438ada2902a36dcbf"
+        "GUID:58e104b97fb3752438ada2902a36dcbf",
+        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
+        "GUID:f55a02e98b01bc849b30d9650ccd8f15"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/MediaPipeUnity/Scripts/SignPredictor.cs

@@ -19,9 +19,9 @@ namespace Mediapipe.Unity.Tutorial
     public class SignPredictor : MonoBehaviour
     {
         /// <summary>
-        /// Reference to the model used in the SignPredictor
+        /// ModelList, used to change model using ModelIndex
         /// </summary>
-        public NNModel model;
+        public ModelList modelList;
 
         /// <summary>
         /// Reference to the model info file
@@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial
         /// </summary>
         private Tensor inputTensor;
 
+        public List<Listener> listeners = new List<Listener>();
+
         /// <summary>
         /// Google Mediapipe setup & run
         /// </summary>
@@ -207,14 +209,21 @@ namespace Mediapipe.Unity.Tutorial
 
 
             keypointManager = new KeypointManager(modelInfoFile);
-
             // check if model exists at path
             //var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
-            worker = model.CreateWorker();
+            worker = modelList.GetCurrentModel().CreateWorker();
 
             StartCoroutine(SignRecognitionCoroutine());
             StartCoroutine(MediapipeCoroutine());
         }
+        /// <summary>
+        /// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called.
+        /// </summary>
+        /// <param name="index">The index of the model to be used</param>
+        public void SetModel(ModelIndex index)
+        {
+            this.modelList.SetCurrentModel(index);
+        }
 
         /// <summary>
         /// Coroutine which executes the mediapipe pipeline
@@ -315,6 +324,10 @@ namespace Mediapipe.Unity.Tutorial
                         learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
                     }
                     //Debug.Log($"prob = [{learnableProbabilities.Aggregate("   ", (t, kv) => $"{t}{kv.Key}:{kv.Value}   ")}]");
+                    foreach(Listener listener in listeners)
+                    {
+                        yield return listener.ProcessIncomingCall();
+                    }
                 }
                 else
                 {

Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef

@@ -7,7 +7,8 @@
         "InterfacesScripts",
         "Unity.TextMeshPro",
         "SpellingBeeScripts",
-        "AccountsScripts"
+        "AccountsScripts",
+        "SignPredictor"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/SpellingBee/Scenes/Game.unity

@@ -1720,6 +1720,10 @@ PrefabInstance:
       propertyPath: m_LocalEulerAnglesHint.z
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+      propertyPath: m_Enabled
+      value: 0
+      objectReference: {fileID: 0}
     - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
       propertyPath: signPredictor
       value: 
@@ -1739,15 +1743,37 @@ RectTransform:
   m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 967164043}
   m_PrefabAsset: {fileID: 0}
---- !u!114 &967164045 stripped
+--- !u!114 &967164046 stripped
 MonoBehaviour:
-  m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 967164043}
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 0}
   m_Enabled: 1
   m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
+  m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &967164047 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 967164043}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &967164048 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 967164043}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
 --- !u!1 &978093274
@@ -3144,7 +3170,7 @@ MonoBehaviour:
       m_Calls:
       - m_Target: {fileID: 1768150807}
         m_TargetAssemblyTypeName: GameController, SpellingBeeScripts
-        m_MethodName: Start
+        m_MethodName: StartController
         m_Mode: 1
         m_Arguments:
           m_ObjectArgument: {fileID: 0}
@@ -3900,7 +3926,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+  modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 1743003084}
@@ -4552,6 +4578,10 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 44fbed5ae228de39b9f727def7578d06, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  feedbackText: {fileID: 967164048}
+  feedbackProgress: {fileID: 967164047}
+  feedbackProgressImage: {fileID: 967164046}
+  signPredictor: {fileID: 1592592444}
   themeList: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
   userList: {fileID: 11400000, guid: 072bec636a40f7e4e93b0ac624a3bda2, type: 2}
   minigame: {fileID: 11400000, guid: 8a087d241d652634eb4f6352267ea7dc, type: 2}
@@ -4562,7 +4592,6 @@ MonoBehaviour:
   timerText: {fileID: 1843239267}
   bonusTimeText: {fileID: 1812475780}
   Scoreboard: {fileID: 862382568}
-  feedback: {fileID: 967164045}
   gameEndedPanel: {fileID: 757133117}
 --- !u!1 &1812475780
 GameObject:

Assets/SpellingBee/Scripts/GameController.cs

@@ -5,8 +5,9 @@ using System.Linq;
 using TMPro;
 using UnityEngine;
 using UnityEngine.UI;
+using DigitalRuby.Tween;
 
-public partial class GameController : MonoBehaviour
+public partial class GameController : AbstractFeedback
 {
     /// <summary>
     /// All of the words that can be used in this session
@@ -136,20 +137,53 @@ public partial class GameController : MonoBehaviour
     /// </summary>
     public Transform Scoreboard;
 
-    /// <summary>
-    /// Accuracy feeback object
-    /// </summary>
-    public Feedback feedback;
-
     /// <summary>
     /// Reference to the gameEnded panel, so we can update its display
     /// </summary>
     public GameObject gameEndedPanel;
 
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
     /// <summary>
     /// Start is called before the first frame update
     /// </summary>
     public void Start()
+    {
+        StartController();
+
+        signPredictor.SetModel(currentTheme.modelIndex);
+        AddSelfAsListener();
+    }
+
+    public void StartController()
     {
         correctLetters = 0;
         incorrectLetters = 0;
@@ -182,29 +216,10 @@ public partial class GameController : MonoBehaviour
         userList.Save();
 
         currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex];
-        feedback.signPredictor.model = currentTheme.model;
+        //feedback.signPredictor.ChangeModel(currentTheme.modelIndex);
         words.AddRange(currentTheme.learnables);
         ShuffleWords();
         NextWord();
-
-        // Set calllbacks
-        feedback.getSignCallback = () =>
-        {
-            if (letterIndex < currentWord.Length)
-            {
-                return currentWord[letterIndex].ToString().ToUpper();
-            }
-            return null;
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
-            if (successful)
-            {
-                AddSeconds(secondsPerLetter);
-            }
-            NextLetter(successful);
-        };
     }
 
     /// <summary>
@@ -453,4 +468,106 @@ public partial class GameController : MonoBehaviour
     {
         yield return new WaitForSecondsRealtime(2);
     }
+
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = GetSign();
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        predictSign(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            predictSign(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+        yield return null;
+    }
+
+    public string GetSign(){
+        if (letterIndex<currentWord.Length){
+            return currentWord[letterIndex].ToString().ToUpper();
+        }
+        return null;
+    }
+
+    public void predictSign(string sign) { 
+        bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
+        if (successful)
+        {
+            AddSeconds(secondsPerLetter);
+        }
+        NextLetter(successful);
+    }
 }

Assets/SpellingBee/Scripts/SpellingBeeScripts.asmdef

@@ -7,7 +7,8 @@
         "GUID:3444c67d5a3a93e5a95a48906078c372",
         "GUID:d0b6b39a21908f94fbbd9f2c196a9725",
         "GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
-        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
+        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
+        "GUID:58e104b97fb3752438ada2902a36dcbf"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

README.md

@@ -1,4 +0,0 @@
-# Unity Application
-
-An overview of all the nessecary information can be found on the following links:
- - [Getting Started](https://gitlab.ilabt.imec.be/wesign/unity-application/-/wikis/home)