Sprint 4

Assets/Courses/Scripts/CoursesController.cs

@@ -0,0 +1,571 @@
+using DigitalRuby.Tween;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using TMPro;
+using UnityEngine;
+using UnityEngine.UI;
+using UnityEngine.Video;
+
+/// <summary>
+/// TemplateCourse scene manager
+/// </summary>
+public class CoursesController : AbstractFeedback
+{
+    /// <summary>
+    /// Reference to the objet holding the title
+    /// </summary>
+    public TMP_Text courseTitle;
+
+    /// <summary>
+    /// The current user
+    /// </summary>
+    private User user;
+
+    /// <summary>
+    /// Current user progress for this course
+    /// </summary>
+    private PersistentDataController.SavedCourseProgress progress = null;
+
+    /// <summary>
+    /// ScriptableObject with list of all courses
+    /// </summary>
+    public CourseList courselist;
+
+    /// <summary>
+    /// Reference to Course ScriptableObject
+    /// </summary>
+    private Course course;
+
+    /// <summary>
+    /// Index of the current word/letter in the course.learnables list
+    /// </summary>
+    private int currentWordIndex = 0;
+
+    /// <summary>
+    /// This holds the amount of words in the course
+    /// </summary>
+    private int maxWords;
+
+    /// <summary>
+    /// The "finished" screen
+    /// </summary>
+    public GameObject ResultPanel;
+
+    /// <summary>
+    /// Reference to the title on the results panel
+    /// </summary>
+    public TMP_Text ResultsTitle;
+
+    /// <summary>
+    /// Reference to the description on the results panel
+    /// </summary>
+    public TMP_Text ResultsDecription;
+
+    /// <summary>
+    /// Button to go back to courses list
+    /// </summary>
+    public Button CoursesButton;
+
+    /// <summary>
+    /// DateTime containint the start moment
+    /// </summary>
+    private DateTime startMoment;
+
+    /// <summary>
+    /// Reference to the timeSpent UI
+    /// </summary>
+    public TMP_Text timeSpent;
+
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    private TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    private Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    private Image feedbackProgressImage;
+
+    public VideoPlayer videoPlayer;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
+    /// <summary>
+    /// Keeps track of what type of panel is currently being used
+    /// </summary>
+    protected int panelId = 0;
+
+    /// <summary>
+    /// Boolean used to check whether the user has already answered the question
+    /// </summary>
+    private bool hasAnswered = false;
+
+    /// <summary>
+    /// Boolean used to check whether  SlideIn animation is playing
+    /// </summary>
+    private bool isNextSignInTransit = false;
+
+    /// <summary>
+    /// Reference to course progress bar
+    /// </summary>
+    public SlicedSlider progressBar;
+
+    /// <summary>
+    /// Reference to the animator of the confetti animation
+    /// </summary>
+    public Animator confettiAnimation;
+
+    /// <summary>
+    /// Panel with video&image prefab
+    /// </summary>
+    public GameObject panelSignWithVideoAndImagePrefab;
+
+    /// <summary>
+    /// Panel with image prefab
+    /// </summary>
+    public GameObject panelSignWithImagePrefab;
+
+    /// <summary>
+    /// Panel with multiplechoice prefab
+    /// </summary>
+    public GameObject panelMultipleChoicePrefab;
+
+    /// <summary>
+    /// Reference to the canvas to put the panels into
+    /// </summary>
+    public Transform canvas;
+
+    /// <summary>
+    /// Reference to the previous panel, 
+    /// so it can be deleted when its done playing its exit animation
+    /// </summary>
+    private GameObject previousPanel = null;
+
+    /// <summary>
+    /// This function is called when the script is initialised.
+    /// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage.
+    /// It takes the correct course from the courselist, using the courseIndex.
+    /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
+    /// Then it sets up the course-screen to display relevant information from the course-scriptable.
+    /// </summary>
+    void Start()
+    {
+        StartCourseController();
+        signPredictor.SetSignsList(GetSignsList());
+        signPredictor.SetModel(course.theme.modelIndex);
+        AddSelfAsListener();
+    }
+
+    /// <summary>
+    /// Fetches all the strings of the signs of the course
+    /// </summary>
+    /// <returns>The signsList that needs to be passed to the signPredictor</returns>
+    private List<string> GetSignsList()
+    {
+        List<string> signsList = new List<string>();
+        foreach (Learnable learnable in course.theme.learnables)
+        {
+            signsList.Add(learnable.name);
+        }
+
+        return signsList;
+    }
+
+    /// <summary>
+    /// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed)
+    /// </summary>
+    public void StartCourseController()
+    {
+        // Setting up course
+        course = courselist.courses[courselist.currentCourseIndex];
+        maxWords = course.theme.learnables.Count;
+
+        // Reload from disk (course may be reset)
+        PersistentDataController.GetInstance().Load();
+        // Create entry in current user for keeping track of progress
+        user = UserList.GetCurrentUser();
+        progress = user.GetCourseProgress(course.index);
+        if (progress == null)
+        {
+            progress = new PersistentDataController.SavedCourseProgress();
+            progress.courseIndex = course.index;
+            int index = 0;
+            foreach (Learnable learnable in course.theme.learnables)
+            {
+                progress.AddLearnable(learnable.name, index++);
+            }
+            user.AddCourseProgress(progress);
+        }
+        UserList.Save();
+        progressBar.fillAmount = progress.progress;
+
+        currentWordIndex = 0;
+        previousPanel = SetupPanel();
+
+        // Hide the result panel
+        ResultPanel.SetActive(false);
+        // Set the startTime
+        startMoment = DateTime.Now;
+    }
+
+    /// <summary>
+    /// Fetch the next sign and its panel type
+    /// </summary>
+    /// <returns>A tuple of {next sign index, panel type}</returns>
+    /// <remarks>
+    /// The different panel types:<br></br>
+    /// 0 : panelSignWithVideoAndImagePrefab<br></br>
+    /// 1 : panelMultipleChoicePrefab<br></br>
+    /// 2 : panelSignWithImagePrefab
+    /// </remarks>
+    private Tuple<int, int> FetchSign()
+    {
+        PersistentDataController.SavedLearnableProgress learnable = progress.GetRandomLearnable();
+        int panelChosen;
+        if (course.theme.modelIndex == ModelIndex.NONE)
+        {
+            // only multiple choice works in preview mode
+            panelChosen = 1;
+        }
+        else if (learnable.progress > 2.0f)
+        {
+            panelChosen = 2;
+        }
+        else if (learnable.progress > 1.0f)
+        {
+            panelChosen = 1;
+        }
+        else
+        {
+            panelChosen = 0;
+        }
+        return Tuple.Create(learnable.index, panelChosen);
+    }
+
+    /// <summary>
+    /// This function is called when the next-sign button is pressed.
+    /// It increased the wordindex and fetches new videos/images if index<max, because then the coure is not fincished yet.
+    /// If the maximum is reached, finishcourse is called to save the "finished" progress to the user.
+    /// </summary>
+    public void NextSign()
+    {
+        // This function is also called (async) when pressing the 'Gebaar overslaan' button,
+        // so check for condition so we don't skip multiple signs
+        if (isNextSignInTransit || maxWords <= progress.completedLearnables)
+            return;
+
+        progress.progress = (float)progress.completedLearnables / (float)maxWords;
+        progressBar.fillAmount = progress.progress;
+
+        // Update UI if course is not finished yet
+        if (progress.completedLearnables < maxWords)
+        {
+            // Set next sign/video/image
+            StartCoroutine(CRNextSign());
+        }
+        // Finish course and record progress
+        if (progress.completedLearnables == maxWords)
+        {
+            FinishCourse();
+        }
+    }
+
+    /// <summary>
+    /// Coroutine for going to the next sign
+    /// </summary>
+    /// <returns></returns>
+    private IEnumerator CRNextSign()
+    {
+        isNextSignInTransit = true;
+        GameObject newPanel = SetupPanel();
+        previousPanel.transform.SetAsFirstSibling();
+        newPanel.GetComponent<Animator>().SetTrigger("Slide Panel In");
+        if (previousPanel != null)
+            previousPanel.GetComponent<Animator>().SetTrigger("Slide Panel Out");
+
+        yield return new WaitForSeconds(1.0f);
+
+        confettiAnimation.ResetTrigger("Display Confetti");
+        GameObject.Destroy(previousPanel);
+        previousPanel = newPanel;
+        hasAnswered = false;
+        isNextSignInTransit = false;
+    }
+
+    /// <summary>
+    /// Setup a new panel
+    /// </summary>
+    /// <returns>Reference to the GameObject of the panel</returns>
+    private GameObject SetupPanel()
+    {
+        (currentWordIndex, panelId) = FetchSign().ToValueTuple();
+        switch (panelId)
+        {
+            case 0:
+                {
+                    GameObject panel = GameObject.Instantiate(panelSignWithVideoAndImagePrefab, canvas);
+                    panel.transform.SetAsFirstSibling();
+
+                    PanelWithVideoAndImage script = panel.GetComponent<PanelWithVideoAndImage>();
+                    script.signs = course.theme.learnables;
+                    script.currentSignIndex = currentWordIndex;
+                    script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
+                    script.videoPlayer = videoPlayer;
+                    feedbackProgress = script.feedbackProgressBar;
+                    feedbackProgressImage = script.feedbackProgressImage;
+                    feedbackText = script.feedbackText;
+                    script.Display();
+                    signPredictor.SwapScreen(script.webcamScreen);
+                    courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\"";
+                    return panel;
+                }
+            case 1:
+                {
+                    GameObject panel = GameObject.Instantiate(panelMultipleChoicePrefab, canvas);
+                    panel.transform.SetAsFirstSibling();
+
+                    PanelMultipleChoice script = panel.GetComponent<PanelMultipleChoice>();
+                    script.signs = course.theme.learnables;
+                    script.currentSignIndex = currentWordIndex;
+                    script.videoPlayer = videoPlayer;
+                    script.courseController = this;
+                    script.progress = progress;
+                    script.isFingerSpelling = course.theme.title == "Handalfabet";
+                    script.Display();
+                    signPredictor.SwapScreen(script.webcamScreen);
+                    courseTitle.text = "Welk gebaar zie je hier?";
+                    return panel;
+                }
+            case 2:
+                {
+                    GameObject panel = GameObject.Instantiate(panelSignWithImagePrefab, canvas);
+                    panel.transform.SetAsFirstSibling();
+
+                    PanelWithImage script = panel.GetComponent<PanelWithImage>();
+                    script.signs = course.theme.learnables;
+                    script.currentSignIndex = currentWordIndex;
+                    script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
+                    feedbackProgress = script.feedbackProgressBar;
+                    feedbackProgressImage = script.feedbackProgressImage;
+                    feedbackText = script.feedbackText;
+                    script.Display();
+                    signPredictor.SwapScreen(script.webcamScreen);
+                    courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\"";
+                    return panel;
+                }
+        }
+        return null;
+    }
+
+    /// <summary>
+    /// finishcourse is called to save the "finished" progress to the user.
+    /// </summary>
+    public void FinishCourse()
+    {
+        // Show the "finished" screen
+        ResultPanel.SetActive(true);
+
+        // Set the correct title
+        ResultsTitle.text = course.title + " voltooid!";
+
+        // Set the correct description
+        ResultsDecription.text = "Goed gedaan! Je kan nu spelletjes spelen met " + course.title + " om verder te oefenen!";
+
+        // Set the total time spent UI
+        TimeSpan time = DateTime.Now - startMoment;
+        timeSpent.text = time.ToString(@"hh\:mm\:ss");
+
+        // Link button
+        CoursesButton.onClick.AddListener(() => { SystemController.GetInstance().BackToPreviousScene(); });
+
+        progress.progress = 1.0f;
+        UserList.Save();
+    }
+
+    /// <summary>
+    /// The updateFunction that is called when new probabilities become available
+    /// </summary>
+    /// <returns></returns>
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Check if the current word index is still in bounds, and if the current panel type is not multiple choice
+        if (currentWordIndex < course.theme.learnables.Count && panelId != 1 && !hasAnswered)
+        {
+            // Get current sign
+            Learnable sign = course.theme.learnables[currentWordIndex];
+            string currentSign = sign.name.ToUpper().Replace(" ", "-");
+
+            // Get the predicted sign
+            if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+                currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+            {
+                //Debug.Log($"{signPredictor.learnableProbabilities.Aggregate("", (t, e) => $"{t}{e.Key}={e.Value}, ")}");
+                float accCurrentSign = signPredictor.learnableProbabilities[currentSign];
+
+                // Get highest predicted sign
+                string predictedSign = signPredictor.learnableProbabilities.Aggregate((a, b) => a.Value > b.Value ? a : b).Key;
+                float accPredictSign = signPredictor.learnableProbabilities[predictedSign];
+                Learnable predSign = course.theme.learnables.Find(l => l.name.ToUpper().Replace(" ", "-") == predictedSign);
+
+                // If there is a feedback-object, we wil change its appearance
+                if (feedbackText != null && feedbackProgressImage != null)
+                {
+                    Color col;
+                    if (accCurrentSign > sign.thresholdPercentage)
+                    {
+                        feedbackText.text = "Goed";
+                        col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
+                    }
+                    else if (accCurrentSign > 0.9 * sign.thresholdPercentage)
+                    {
+                        feedbackText.text = "Bijna";
+                        col = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f);
+                    }
+                    else if (accPredictSign > predSign.thresholdPercentage)
+                    {
+                        feedbackText.text = $"Verkeerde gebaar: '{predSign.name}'";
+                        col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
+                        accCurrentSign = 0.0f;
+                    }
+                    else
+                    {
+                        feedbackText.text = $"Detecteren ...";
+                        col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
+                        //accCurrentSign = 0.0f;
+                    }
+
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+
+                    // Tween the feedback-bar
+                    float oldValue = feedbackProgress.value;
+                    // use an exponential scale 
+                    float newValue = Mathf.Exp(4 * (Mathf.Clamp(accCurrentSign / sign.thresholdPercentage, 0.0f, 1.0f) - 1.0f));
+                    feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                    {
+                        if (feedbackProgress != null)
+                        {
+                            feedbackProgress.value = t.CurrentValue;
+                        }
+                    });
+                }
+
+                // The internal logic for the courses
+                if (accPredictSign > sign.thresholdPercentage)
+                {
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        NextSignIfCorrect(currentSign, predictedSign);
+                        timer = DateTime.Now;
+                        previousIncorrectSign = null;
+                        predictedSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (predictedSign != null && currentSign != null &&
+                            (DateTime.Now - timer).TotalSeconds > 2.0f)
+                        {
+                            NextSignIfCorrect(currentSign, predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                }
+            }
+            else if (feedbackProgress != null)
+            {
+                feedbackProgress.value = 0.0f;
+            }
+        }
+    }
+
+    /// <summary>
+    /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
+    /// </summary>
+    /// <param name="predicted"></param>
+    public void NextSignIfCorrect(string current, string predicted)
+    {
+        if (!hasAnswered)
+        {
+            if (current == predicted)
+            {
+                hasAnswered = true;
+                var p = progress.learnables.Find((l) => l.name.ToUpper().Replace(" ", "-") == predicted);
+                progress.UpdateLearnable(p.name, 1.5f);
+                confettiAnimation.SetTrigger("Display Confetti");
+                StartCoroutine(WaitNextSign());
+            }
+            else
+            {
+                // currently ignore wrong signs as "J" doesn't work well enough
+            }
+        }
+
+    }
+
+    private IEnumerator WaitNextSign()
+    {
+        // Wait for 0.75 seconds
+        yield return new WaitForSeconds(0.75f);
+        NextSign();
+    }
+
+    /// <summary>
+    /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
+    /// </summary>
+    /// <param name="predicted"></param>
+    public void NextSignMultipleChoice(string current, string predicted)
+    {
+        if (!hasAnswered)
+        {
+            hasAnswered = true;
+            if (current == predicted)
+            {
+                progress.UpdateLearnable(predicted, 1.5f);
+                confettiAnimation.SetTrigger("Display Confetti");
+            }
+            else
+            {
+                progress.UpdateLearnable(predicted, -1.0f);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Callback for the 'back' button
+    /// </summary>
+    public void ReturnToActivityScreen()
+    {
+        UserList.Save();
+        SystemController.GetInstance().BackToPreviousScene();
+    }
+}