Updated diffusion section of thesis + adding metrics to diffusion experiments

src/trainers/diffusion_trainer.py

@@ -240,7 +240,7 @@ class DiffusionTrainer:
         self.model = torch.load("checkpoint.pt")
         self.model.to(self.device)
 
-        _, generated_sampels = self.test(test_loader, None, task)
+        _, generated_sampels = self.test(test_loader, -1, task)
         # self.policy_evaluator.plot_profits_table()
 
         optimal_penalty, profit, charge_cycles = (
@@ -392,6 +392,9 @@ class DiffusionTrainer:
         self, data_loader: torch.utils.data.DataLoader, epoch: int, task: Task = None
     ):
         all_crps = []
+        all_inversed_crps = []
+        all_inversed_mae = []
+        all_inversed_mse = []
         generated_samples = {}
         for inputs, targets, idx_batch in data_loader:
             inputs, targets = inputs.to(self.device), targets.to(self.device)
@@ -413,12 +416,58 @@ class DiffusionTrainer:
             crps = crps_from_samples(samples_batched, targets)
             crps_mean = crps.mean(axis=1)
 
+            if epoch == -1:
+                inversed_samples_batched = self.data_processor.inverse_transform(
+                    samples_batched
+                )
+                inversed_targets = self.data_processor.inverse_transform(targets)
+
+                inversed_crps = crps_from_samples(
+                    inversed_samples_batched, inversed_targets
+                )
+                inversed_crps_mean = inversed_crps.mean(axis=1)
+                all_inversed_crps.extend(inversed_crps_mean.tolist())
+
+                inversed_expanded_targets = inversed_targets.unsqueeze(1).expand(
+                    -1, number_of_samples, -1
+                )
+
+                inversed_mae = np.abs(
+                    inversed_samples_batched - inversed_expanded_targets
+                )
+                inversed_mae_mean = inversed_mae.mean()
+                all_inversed_mae.extend(inversed_mae_mean.tolist())
+
+                inversed_mse = np.square(
+                    inversed_samples_batched - inversed_expanded_targets
+                )
+                inversed_mse_mean = inversed_mse.mean()
+                all_inversed_mse.extend(inversed_mse_mean.tolist())
+
             # add all values from crps_mean to all_crps
             all_crps.extend(crps_mean.tolist())
 
         all_crps = np.array(all_crps)
         mean_crps = all_crps.mean()
 
+        if epoch == -1:
+            all_inversed_crps = np.array(all_inversed_crps)
+            mean_inversed_crps = all_inversed_crps.mean()
+
+            task.get_logger().report_single_value(
+                name="test_CRPSLoss", value=mean_inversed_crps
+            )
+
+            mean_inversed_mae = np.array(all_inversed_mae).mean()
+            task.get_logger().report_single_value(
+                name="test_MSELoss", value=mean_inversed_mae
+            )
+
+            mean_inversed_mse = np.array(all_inversed_mse).mean()
+            task.get_logger().report_single_value(
+                name="test_L1Loss", value=mean_inversed_mse
+            )
+
         if self.best_score is None or mean_crps < self.best_score:
             self.save_checkpoint(mean_crps, task, epoch)
 

src/training_scripts/autoregressive_quantiles.py

@@ -5,7 +5,7 @@ clearml_helper = ClearMLHelper(project_name="Thesis/NrvForecast")
 task = clearml_helper.get_task(
     task_name="AQR: GRU (8 - 512) + Load + PV + Wind + NP + QE (dim 5)"
 )
-task.execute_remotely(queue_name="default", exit_process=True)
+# task.execute_remotely(queue_name="default", exit_process=True)
 
 from src.policies.PolicyEvaluator import PolicyEvaluator
 from src.policies.simple_baseline import BaselinePolicy, Battery
@@ -24,7 +24,6 @@ from torch.nn import MSELoss, L1Loss
 import torch.nn as nn
 from src.models.time_embedding_layer import TimeEmbedding, TrigonometricTimeEmbedding
 
-
 #### Data Processor ####
 data_config = DataConfig()