WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-131-Feedback-REfactor

Browse Source
This commit is contained in:
Jerome Coudron
2023-04-02 12:27:59 +00:00
committed by Dries Van Schuylenbergh
parent b955d2164c
commit a808e73a29
27 changed files with 663 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
Assets/Courses/Scenes/CourseScreen.unity
View File

@@ -1273,7 +1273,6 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
screen: {fileID: 378145456}
@@ -1529,6 +1528,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3}
m_Name:
m_EditorClassIdentifier:
signPredictor: {fileID: 883853268}
previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
feedbackProgressBar: {fileID: 4318122121437849759}
previewMessage: {fileID: 2070775951}
@@ -1544,7 +1544,9 @@ MonoBehaviour:
ResultsDecription: {fileID: 100123246}
CoursesButton: {fileID: 839294691}
timeSpent: {fileID: 77614869}
feedback: {fileID: 1714882683}
feedbackText: {fileID: 4318122121437849762}
feedbackProgress: {fileID: 4318122121437849761}
feedbackProgressImage: {fileID: 4318122121437849760}
--- !u!4 &1122267057
Transform:
m_ObjectHideFlags: 0
@@ -2284,17 +2286,6 @@ RectTransform:
m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
--- !u!114 &1714882683 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122121437849759}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!1 &1773033262
GameObject:
m_ObjectHideFlags: 0
@@ -2897,3 +2888,36 @@ GameObject:
m_CorrespondingSourceObject: {fileID: 4318122119930585319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
--- !u!114 &4318122121437849760 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!114 &4318122121437849761 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!114 &4318122121437849762 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
m_Name:
m_EditorClassIdentifier:

4
Assets/Courses/Scripts/CourseScripts.asmdef
View File

@@ -5,8 +5,8 @@
"Unity.TextMeshPro",
"AccountsScripts",
"InterfacesScripts",
"SignPredictor",
"Unity.Barracuda"
"Tween",
"SignPredictor"
],
"includePlatforms": [],
"excludePlatforms": [],

174
Assets/Courses/Scripts/CoursesController.cs
View File

@@ -1,6 +1,7 @@
using DigitalRuby.Tween;
using System;
using System.Collections;
using TMPro;
using Unity.Barracuda;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.Video;
@@ -8,13 +9,10 @@ using UnityEngine.Video;
/// <summary>
/// TemplateCourse scene manager
/// </summary>
public class CoursesController : MonoBehaviour
public class CoursesController : AbstractFeedback
{
// vvv TEMPORARY STUFF vvv
public NNModel previewModel;
public GameObject feedbackProgressBar;
public GameObject previewMessage;
// ^^^ TEMPORARY STUFF ^^^
/// <summary>
/// Reference to instructional video player
@@ -112,10 +110,37 @@ public class CoursesController : MonoBehaviour
/// </summary>
public TMP_Text timeSpent;
/// <summary>
/// Reference to the feedback script on the Feedback prefab
/// Reference to the feedback field
/// </summary>
public Feedback feedback;
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
/// </summary>
protected DateTime timer;
/// <summary>
/// Current predicted sign
/// </summary>
protected string predictedSign = null;
/// <summary>
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
/// </summary>
protected string previousIncorrectSign = null;
/// <summary>
/// This function is called when the script is initialised.
@@ -124,19 +149,25 @@ public class CoursesController : MonoBehaviour
/// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
/// Then it sets up the course-screen to display relevant information from the course-scriptable.
/// </summary>
void Awake()
void Start()
{
StartCourseController();
signPredictor.SetModel(course.theme.modelIndex);
AddSelfAsListener();
}
/// <summary>
/// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed)
/// </summary>
public void StartCourseController()
{
// Setting up course
course = courselist.courses[courselist.currentCourseIndex];
feedback.signPredictor.ChangeModel(course.theme.modelIndex);
maxWords = course.theme.learnables.Count;
// vvv TEMPORARY STUFF vvv
// Show preview messages if there is no model
feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE);
previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE);
// Instead, the NONE-modelIndex points to Fingerspelling, which gives the same result
//feedback.signPredictor.model = previewModel;
// ^^^ TEMPORARY STUFF ^^^
// Create entry in current user for keeping track of progress
userList.Load();
@@ -164,23 +195,6 @@ public class CoursesController : MonoBehaviour
ResultPanel.SetActive(false);
// Set the startTime
startMoment = DateTime.Now;
// Set callbacks
feedback.getSignCallback = () =>
{
if (currentWordIndex < course.theme.learnables.Count)
{
return course.theme.learnables[currentWordIndex].name;
}
return null;
};
feedback.predictSignCallback = (sign) =>
{
if (sign == course.theme.learnables[currentWordIndex].name)
{
NextSign();
}
};
}
/// <summary>
@@ -287,4 +301,104 @@ public class CoursesController : MonoBehaviour
progress.AddOrUpdate<float>("courseProgress", 1f);
userList.Save();
}
/// <summary>
/// The updateFunction that is called when new probabilities become available
/// </summary>
/// <returns></returns>
protected override IEnumerator UpdateFeedback()
{
// Get current sign
string currentSign = course.theme.learnables[currentWordIndex].name;
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null)
{
if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
CheckEquality(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
CheckEquality(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if (feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
/// <summary>
/// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
/// </summary>
/// <param name="predicted"></param>
private void CheckEquality(string predicted)
{
if (predicted == course.theme.learnables[currentWordIndex].name)
{
NextSign();
}
}
}