WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-143 "Feedback courses"

Browse Source
This commit is contained in:
Helena Van Breugel
2023-04-04 18:24:17 +00:00
committed by Jerome Coudron
parent c20cd89c3a
commit c358ac59e4
115 changed files with 11331 additions and 1116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

317
Assets/Courses/Scripts/CoursesController.cs
View File

@@ -1,6 +1,7 @@
using DigitalRuby.Tween;
using System;
using System.Collections;
using System.Linq;
using TMPro;
using UnityEngine;
using UnityEngine.UI;
@@ -11,33 +12,7 @@ using UnityEngine.Video;
/// </summary>
public class CoursesController : AbstractFeedback
{
public GameObject feedbackProgressBar;
public GameObject previewMessage;
/// <summary>
/// Reference to instructional video player
/// </summary>
public VideoPlayer player;
/// <summary>
/// Reference to pause button
/// </summary>
public Button button;
/// <summary>
/// Reference to sprite for the pause button
/// </summary>
public Sprite pauseSprite;
/// <summary>
/// Reference to the image for displaying the current words sprite
/// </summary>
public Image wordImage;
/// <summary>
/// Reference to the text object for displaying the current word
/// </summary>
public TMP_Text title;
public TMP_Text courseTitle;
/// <summary>
/// The current user
@@ -105,21 +80,22 @@ public class CoursesController : AbstractFeedback
/// </summary>
public TMP_Text timeSpent;
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
private TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
private Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
private Image feedbackProgressImage;
public VideoPlayer videoPlayer;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
@@ -136,6 +112,12 @@ public class CoursesController : AbstractFeedback
/// </summary>
protected string previousIncorrectSign = null;
public Animator confettiAnimation;
public GameObject panelSignWithVideoAndImagePrefab;
public GameObject panelSignWithImagePrefab;
public GameObject panelMultipleChoicePrefab;
public Transform canvas;
private GameObject previousPanel = null;
/// <summary>
/// This function is called when the script is initialised.
@@ -147,9 +129,10 @@ public class CoursesController : AbstractFeedback
void Start()
{
StartCourseController();
signPredictor.ChangeModel(course.theme.modelIndex);
signPredictor.SetModel(course.theme.modelIndex);
AddSelfAsListener();
}
/// <summary>
/// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed)
/// </summary>
@@ -159,10 +142,6 @@ public class CoursesController : AbstractFeedback
course = courselist.courses[courselist.currentCourseIndex];
maxWords = course.theme.learnables.Count;
// Show preview messages if there is no model
feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE);
previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE);
// Create entry in current user for keeping track of progress
//PersistentDataController pdc = PersistentDataController.GetInstance();
//pdc.Load();
@@ -175,15 +154,10 @@ public class CoursesController : AbstractFeedback
user.AddCourseProgress(progress);
}
UserList.Save();
courseTitle.text = course.title;
// Force the videoplayer to add bars to preserve aspect ratio
player.aspectRatio = VideoAspectRatio.FitInside;
// Setup UI
button.image.sprite = pauseSprite;
title.text = course.title;
NextVideo();
NextImage();
currentWordIndex = 0;
previousPanel = SetupPanel();
// Hide the result panel
ResultPanel.SetActive(false);
@@ -191,53 +165,29 @@ public class CoursesController : AbstractFeedback
startMoment = DateTime.Now;
}
/// <summary>
/// This function uses the word_i integer to grab the correct video from the course.learnabels.
/// When it has this video, it will load it into the videoplayer and set it to start.
/// </summary>
private void NextVideo()
private Tuple<int, int> FetchSign()
{
player.clip = course.theme.learnables[currentWordIndex].clip;
// This loads first frame, so that it can be used as a sort-of preview for the video
player.Play();
// As the video will start playiing -> hide button
Color col = button.image.color;
col.a = 0;
button.image.color = col;
}
// TODO: @Tibe here you need to provide other signs and there question method
/// <summary>
/// This function uses the word_i integer to grab the correct image from the course.learnabels.
/// Then it simply loads it into wordImage so that it can be displayed.
/// </summary>
private void NextImage()
{
wordImage.sprite = course.theme.learnables[currentWordIndex].image;
}
/**************************
* TODO: @Tibe
*
* In deze functie beslist welk panel je nu nodig hebt
* Momenteel doe ik gwn iets om te wisselen tussen de twee (moet zeker weg want je begint ALTIJD met een imageANDvideo
*
* Je ziet zelf maar hoe groot je de sets van woorden maakt, om de 5 ofzo
* Altijd eerst video and image, nadien kan je afwisselen, mag random
*
* ALSO:
* Hiervoor moet ge bij Dries zijn, maar man is verdwenen. (100 jaar gingen voorbij en mijn broer en ik vonden een nieuwe oorzaak voor hoofdpijn)
* Progress gaat ook aangepast moeten worden, als een user terugkeert tijdens een course en later hervat ga je moeten weten welke set die zat
* Zeker als ge woorden in een andere volgorde zou willen doen, gaat ge echt nog aan uw 40u geraken :)
* --> dat gaat bijgehouden worden in een van die user files. (Stalk Dries indien nodig, voor andere zaken kan je mij ook vragen stellen)
*
* *************************/
/// <summary>
/// This function is called when the pause-button is pressed on the video.
/// It switches between playing and pausing the video.
/// It then makes the button invisible when the video is playing, or visible when it's paused.
/// </summary>
public void Pause()
{
if (!player.isPlaying)
{
// Play video and hide button
player.Play();
Color col = button.image.color;
col.a = 0;
button.image.color = col;
}
else
{
// Pause video and show button
player.Pause();
Color col = button.image.color;
col.a = 255;
button.image.color = col;
}
int panelChosen = currentWordIndex % 2 + 1;
return Tuple.Create(currentWordIndex, panelChosen);
}
/// <summary>
@@ -250,6 +200,8 @@ public class CoursesController : AbstractFeedback
// If the currentindex >= maxwords, it indicated that the course is already finished, running the next code is the meaningless.
if (currentWordIndex >= maxWords) { return; }
confettiAnimation.SetTrigger("Display Confetti");
// Goto the next word/letter
currentWordIndex++;
@@ -261,8 +213,8 @@ public class CoursesController : AbstractFeedback
// Update UI if course is not finished yet
if (currentWordIndex < maxWords)
{
NextVideo();
NextImage();
// Set next sign/video/image
StartCoroutine(CRNextSign());
}
// Finish course and record progress
else
@@ -271,6 +223,72 @@ public class CoursesController : AbstractFeedback
}
}
private IEnumerator CRNextSign()
{
GameObject newPanel = SetupPanel();
previousPanel.transform.SetAsFirstSibling();
newPanel.GetComponent<Animator>().SetTrigger("Slide Panel");
yield return new WaitForSeconds(1.0f);
confettiAnimation.ResetTrigger("Display Confetti");
GameObject.Destroy(previousPanel);
previousPanel = newPanel;
}
private GameObject SetupPanel()
{
int panelId;
(currentWordIndex, panelId) = FetchSign().ToValueTuple();
switch (panelId)
{
case 0: return null; // TODO: @Tibe multiple choice setup
/**************************
* TODO: @Tibe
*
* Hier moet de panelMultipleChoice worden aangemaakt
* Geef publieke dingen mee aan uw script.
* Kan je eventueel zelf aanpassen,
* naargelang hoe je het wilt implementeren
*
* *************************/
case 1:
{
GameObject panel = GameObject.Instantiate(panelSignWithImagePrefab, canvas);
panel.transform.SetAsFirstSibling();
PanelWithImage script = panel.GetComponent<PanelWithImage>();
script.signs = course.theme.learnables;
script.currentSignIndex = currentWordIndex;
script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
feedbackProgress = script.feedbackProgressBar;
feedbackProgressImage = script.feedbackProgressImage;
feedbackText = script.feedbackText;
script.Display();
signPredictor.SwapScreen(script.webcamScreen);
return panel;
}
case 2:
{
GameObject panel = GameObject.Instantiate(panelSignWithVideoAndImagePrefab, canvas);
panel.transform.SetAsFirstSibling();
PanelWithVideoAndImage script = panel.GetComponent<PanelWithVideoAndImage>();
script.signs = course.theme.learnables;
script.currentSignIndex = currentWordIndex;
script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
script.videoPlayer = videoPlayer;
feedbackProgress = script.feedbackProgressBar;
feedbackProgressImage = script.feedbackProgressImage;
feedbackText = script.feedbackText;
script.Display();
signPredictor.SwapScreen(script.webcamScreen);
return panel;
}
}
return null;
}
/// <summary>
/// finishcourse is called to save the "finished" progress to the user.
/// </summary>
@@ -295,67 +313,81 @@ public class CoursesController : AbstractFeedback
progress.progress = 1.0f;
UserList.Save();
}
/// <summary>
/// The updateFunction that is called when new probabilities become available
/// </summary>
/// <returns></returns>
protected override IEnumerator UpdateFeedback()
{
// Get current sign
string currentSign = course.theme.learnables[currentWordIndex].name;
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
if (currentWordIndex < course.theme.learnables.Count)
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null)
// Get current sign
Learnable sign = course.theme.learnables[currentWordIndex];
string currentSign = sign.name;
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
if (accuracy > 0.90)
float accCurrentSign = signPredictor.learnableProbabilities[currentSign];
// Get highest predicted sign
string predictedSign = signPredictor.learnableProbabilities.Aggregate((a, b) => a.Value > b.Value ? a : b).Key;
float accPredictSign = signPredictor.learnableProbabilities[predictedSign];
Learnable predSign = course.theme.learnables.Find(l => l.name == predictedSign);
if (feedbackText != null && feedbackProgressImage != null)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
Color col;
if (accCurrentSign > sign.thresholdPrecentage)
{
feedbackText.text = "Goed";
col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
}
else if (accCurrentSign > 0.9 * sign.thresholdPrecentage)
{
feedbackText.text = "Bijna";
col = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f);
}
else if (accPredictSign > predSign.thresholdPrecentage)
{
feedbackText.text = $"Verkeerde gebaar: '{predictedSign}'";
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
accCurrentSign = 0.0f;
}
else
{
feedbackText.text = $"Detecteren ...";
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
//accCurrentSign = 0.0f;
}
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accCurrentSign - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
feedbackProgress.value = t.CurrentValue;
}
});
}
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
if (accPredictSign > sign.thresholdPrecentage)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
CheckEquality(predictedSign);
NextSignIfCorrect(currentSign, predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
predictedSign = null;
}
// Incorrect sign
else
@@ -365,34 +397,33 @@ public class CoursesController : AbstractFeedback
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
else if (predictedSign != null && currentSign != null &&
(DateTime.Now - timer).TotalSeconds > 2.0f)
{
CheckEquality(predictedSign);
NextSignIfCorrect(currentSign, predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
else if (feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
else if (feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
/// <summary>
/// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
/// </summary>
/// <param name="predicted"></param>
private void CheckEquality(string predicted)
private void NextSignIfCorrect(string current, string predicted)
{
if (predicted == course.theme.learnables[currentWordIndex].name)
{
if (current == predicted)
NextSign();
}
// TODO: @Tibe cache the incorrect values here
}
}