WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Feedback SpellingBee

Browse Source
This commit is contained in:
Helena Van Breugel
2023-04-05 13:40:54 +00:00
committed by Jelle De Geest
parent e6b0cbd596
commit 2a7dd16f5f
5 changed files with 271 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
Assets/SpellingBee/Scripts/SpellingBeeController.cs
View File

@@ -50,6 +50,11 @@ public partial class SpellingBeeController : AbstractFeedback
/// </summary>
private bool gameEnded;
/// <summary>
/// List of learnables to get the threshold for the letters
/// </summary>
public Theme fingerspelling;
/// <summary>
/// Amount of seconds user gets per letter of the current word
/// Set to 1 for testing; should be increased later
@@ -389,12 +394,12 @@ public partial class SpellingBeeController : AbstractFeedback
if (successful)
{
correctLetters++;
letters[letterIndex].GetComponent<Image>().color = Color.green;
letters[letterIndex].GetComponent<Image>().color = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
}
else
{
incorrectLetters++;
letters[letterIndex].GetComponent<Image>().color = new Color(0.5f, 0.0f, 0.0f);
letters[letterIndex].GetComponent<Image>().color = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
}
do
@@ -405,7 +410,7 @@ public partial class SpellingBeeController : AbstractFeedback
// Change the color of the next letter or change to new word
if (letterIndex < currentWord.Length)
{
letters[letterIndex].GetComponent<Image>().color = Color.yellow;
letters[letterIndex].GetComponent<Image>().color = new Color(0x9f / 255.0f, 0xe7 / 255.0f, 0xf5 / 255.0f);
}
else
{
@@ -451,7 +456,7 @@ public partial class SpellingBeeController : AbstractFeedback
// Dynamically load appearance
char c = Char.ToUpper(word[i]);
Image background = instance.GetComponent<Image>();
background.color = i == 0 ? Color.yellow : c != ' ' ? Color.red : Color.clear;
background.color = i == 0 ? new Color(0x9f / 255.0f, 0xe7 / 255.0f, 0xf5 / 255.0f) : Color.clear;
TMP_Text txt = instance.GetComponentInChildren<TMP_Text>();
txt.text = Char.ToString(c);
}
@@ -467,6 +472,17 @@ public partial class SpellingBeeController : AbstractFeedback
yield return new WaitForSecondsRealtime(2);
}
/// <summary>
/// Get the threshold for a given sign
/// </summary>
/// <param name="sign"></param>
/// <returns></returns>
private float GetTresholdPercentage(string sign)
{
Learnable letter = fingerspelling.learnables.Find(l => l.name == sign);
return letter.thresholdPercentage;
}
/// <summary>
/// The updateFunction that is called when new probabilities become available
/// </summary>
@@ -479,32 +495,45 @@ public partial class SpellingBeeController : AbstractFeedback
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
float accCurrentSign = signPredictor.learnableProbabilities[currentSign];
float thresholdCurrentSign = GetTresholdPercentage(currentSign);
// Get highest predicted sign
string predictedSign = signPredictor.learnableProbabilities.Aggregate((a, b) => a.Value > b.Value ? a : b).Key;
float accPredictSign = signPredictor.learnableProbabilities[predictedSign];
float thresholdPredictedSign = GetTresholdPercentage(predictedSign);
if (feedbackText != null && feedbackProgressImage != null)
{
if (accuracy > 0.90)
Color col;
if (accCurrentSign > thresholdCurrentSign)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
}
else if (accuracy > 0.80)
else if (accCurrentSign > 0.9 * thresholdCurrentSign)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
col = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f);
}
else if (accPredictSign > thresholdPredictedSign)
{
feedbackText.text = $"Verkeerde gebaar: '{predictedSign}'";
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
accCurrentSign = 0.0f;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
}
feedbackText.color = col;
feedbackProgressImage.color = col;
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
float newValue = Mathf.Exp(4 * (accCurrentSign - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
@@ -514,39 +543,35 @@ public partial class SpellingBeeController : AbstractFeedback
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
if (accPredictSign > thresholdPredictedSign)
{
if (kv.Value > 0.90)
// Correct sign
if (predictedSign == currentSign)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
yield return new WaitForSeconds(1.0f);
PredictSign(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
yield return new WaitForSeconds(1.0f);
predictSign(predictedSign);
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
PredictSign(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
predictSign(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if (feedbackProgress != null)
@@ -572,7 +597,7 @@ public partial class SpellingBeeController : AbstractFeedback
/// Function to confirm your prediction and check if it is correct.
/// </summary>
/// <param name="sign"></param>
public void predictSign(string sign)
public void PredictSign(string sign)
{
bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
if (successful)