Added diffusion validation set

src/data/preprocessing.py

@@ -50,8 +50,8 @@ class DataProcessor:
             datetime(year=2022, month=11, day=30, tzinfo=pytz.UTC),
         )
         self.val_range = (
-            datetime(year=2022, month=10, day=1, tzinfo=pytz.UTC),
-            datetime(year=2022, month=11, day=30, tzinfo=pytz.UTC),
+            datetime(year=2022, month=11, day=1, tzinfo=pytz.UTC),
+            datetime(year=2022, month=12, day=30, tzinfo=pytz.UTC),
         )
         self.test_range = (datetime(year=2023, month=1, day=1, tzinfo=pytz.UTC), np.inf)
 
@@ -297,10 +297,10 @@ class DataProcessor:
     ):
         val_df = self.all_features.copy()
 
-        if self.test_range[0] != -np.inf:
+        if self.val_range[0] != -np.inf:
             val_df = val_df[(val_df["datetime"] >= self.val_range[0])]
 
-        if self.test_range[1] != np.inf:
+        if self.val_range[1] != np.inf:
             val_df = val_df[(val_df["datetime"] <= self.val_range[1])]
 
         if transform:

src/policies/PolicyEvaluator.py

@@ -15,7 +15,17 @@ class PolicyEvaluator:
         self.baseline_policy = baseline_policy
 
         self.ipc = ImbalancePriceCalculator(data_path="")
+
         self.dates = baseline_policy.test_data["DateTime"].dt.date.unique()
+
+        # also add dates from last 2 months of 2023
+        self.dates = np.append(
+            self.dates,
+            pd.date_range(
+                start="2022-11-01", end="2022-12-31", freq="D"
+            ).to_pydatetime(),
+        )
+
         self.dates = pd.to_datetime(self.dates)
 
         ### Load Imbalance Prices ###
@@ -116,6 +126,10 @@ class PolicyEvaluator:
             # Calculate the gradient (difference) between the simulated and target charge cycles
             gradient = simulated_charge_cycles - target_charge_cycles
 
+            if abs(gradient) < tolerance:
+                print(f"Optimal penalty found after {iteration+1} iterations")
+                break
+
             # Optionally, adjust learning rate based on the change of gradient direction to avoid oscillation
             if previous_gradient is not None and gradient * previous_gradient < 0:
                 learning_rate *= learning_rate_decay
@@ -129,9 +143,7 @@ class PolicyEvaluator:
             previous_gradient = gradient
 
             # Check if the charge cycles are close enough to the target
-            if abs(gradient) < tolerance:
-                print(f"Optimal penalty found after {iteration+1} iterations")
-                break
+            
 
         else:
             print(
@@ -218,7 +230,7 @@ class PolicyEvaluator:
                 raise KeyboardInterrupt
 
             except Exception as e:
-                print(e)
+                # print(e)
                 pass
 
         self.profits = pd.DataFrame(
@@ -243,6 +255,8 @@ class PolicyEvaluator:
 
         loggings = []
 
+        total_dates = 0
+
         for date in tqdm(self.dates):
             try:
                 (
@@ -272,15 +286,18 @@ class PolicyEvaluator:
                 }
 
                 loggings.append(new_info)
+                total_dates += 1
 
             except KeyboardInterrupt:
                 print("Interrupted")
                 raise KeyboardInterrupt
 
             except Exception as e:
-                print(e)
+                # print(e)
                 pass
 
+        print(f"Total Evaluated Dates: {total_dates}")
+
         if log_metrics:
             log_df = pd.DataFrame(loggings)
 

src/trainers/diffusion_trainer.py

@@ -98,7 +98,7 @@ class DiffusionTrainer:
         self.best_score = None
         self.policy_evaluator = policy_evaluator
 
-        self.prev_optimal_penalty = 0
+        self.prev_optimal_penalty = 600
 
     def noise_time_series(self, x: torch.tensor, t: int):
         """Add noise to time series
@@ -198,6 +198,8 @@ class DiffusionTrainer:
 
         early_stopping = 0
         best_crps = None
+        best_profit = None
+        best_charge_cycles = None
 
         if task:
             self.init_clearml_task(task)
@@ -206,6 +208,12 @@ class DiffusionTrainer:
             predict_sequence_length=self.ts_length, validation=True
         )
 
+        # val loader total samples
+        val_loader_total_samples = len(val_loader.dataset)
+        print("Train Loader Total Samples: ", len(train_loader.dataset))
+        print(f"Val Loader Total Samples: {val_loader_total_samples}")
+        print(f"Test Loader Total Samples: {len(test_loader.dataset)}")
+
         train_sample_indices = self.random_samples(train=True, num_samples=5)
         test_sample_indices = self.random_samples(train=False, num_samples=5)
 
@@ -229,16 +237,16 @@ class DiffusionTrainer:
 
             running_loss /= len(train_loader.dataset)
 
-            if epoch % 75 == 0 and epoch != 0:
-                crps, _ = self.test(val_loader, epoch, task)
+            if epoch % 30 == 0 and epoch != 0:
+                crps, profit, charge_cycles, _  = self.test(val_loader, epoch, task)
 
-                if best_crps is None or crps < best_crps:
-                    best_crps = crps
+                if best_profit is None or profit > best_profit:
+                    best_profit = profit
                     early_stopping = 0
                 else:
                     early_stopping += 1
 
-                if early_stopping > 5:
+                if early_stopping > 20:
                     break
 
             if task:
@@ -249,7 +257,7 @@ class DiffusionTrainer:
                     value=loss.item(),
                 )
 
-                if epoch % 150 == 0 and epoch != 0:
+                if epoch % 300 == 0 and epoch != 0:
                     self.debug_plots(
                         task, True, train_loader, train_sample_indices, epoch
                     )
@@ -580,15 +588,12 @@ class DiffusionTrainer:
                 name="test_MSELoss", value=mean_inversed_mse
             )
 
-        if self.best_score is None or mean_crps < self.best_score:
-            self.save_checkpoint(mean_crps, task, epoch)
-
         if task:
             task.get_logger().report_scalar(
                 title="CRPS", series="val", value=mean_crps, iteration=epoch
             )
 
-        if self.policy_evaluator:
+        if self.policy_evaluator and epoch != -1:
             _, val_loader, _ = self.data_processor.get_dataloaders(
                 predict_sequence_length=self.ts_length,
                 full_day_skip=True,
@@ -600,10 +605,10 @@ class DiffusionTrainer:
                     idx_samples=generated_samples,
                     test_loader=val_loader,
                     initial_penalty=self.prev_optimal_penalty,
-                    target_charge_cycles=283,
-                    initial_learning_rate=1,
+                    target_charge_cycles=58*400/356,
+                    initial_learning_rate=20,
                     max_iterations=50,
-                    tolerance=1,
+                    tolerance=0.3,
                     iteration=epoch,
                 )
             )
@@ -612,22 +617,28 @@ class DiffusionTrainer:
 
             task.get_logger().report_scalar(
                 title="Optimal Penalty",
-                series="test",
+                series="val",
                 value=optimal_penalty,
                 iteration=epoch,
             )
 
             task.get_logger().report_scalar(
-                title="Optimal Profit", series="test", value=profit, iteration=epoch
+                title="Optimal Profit", series="val", value=profit, iteration=epoch
             )
 
             task.get_logger().report_scalar(
                 title="Optimal Charge Cycles",
-                series="test",
+                series="val",
                 value=charge_cycles,
                 iteration=epoch,
             )
 
+            if self.best_score is None or profit > self.best_score:
+                self.save_checkpoint(profit, task, epoch)
+
+
+            return mean_crps, profit, charge_cycles, generated_samples
+
         return mean_crps, generated_samples
 
     def save_checkpoint(self, val_loss, task, iteration: int):

src/training_scripts/diffusion_training.py

@@ -2,9 +2,9 @@ from src.utils.clearml import ClearMLHelper
 
 clearml_helper = ClearMLHelper(project_name="Thesis/NrvForecast")
 task = clearml_helper.get_task(
-    task_name="Diffusion Training: hidden_sizes=[256, 256] (100 steps), lr=0.0001, time_dim=8",
+    task_name="Diffusion Training: hidden_sizes=[512, 512] (100 steps), lr=0.0001, time_dim=8",
 )
-task.execute_remotely(queue_name="default", exit_process=True)
+# task.execute_remotely(queue_name="default", exit_process=True)
 
 from src.models import *
 from src.losses import *
@@ -19,16 +19,16 @@ from src.policies.PolicyEvaluator import PolicyEvaluator
 data_config = DataConfig()
 data_config.NRV_HISTORY = True
 
-data_config.LOAD_HISTORY = False
-data_config.LOAD_FORECAST = False
+data_config.LOAD_HISTORY = True
+data_config.LOAD_FORECAST = True
 
-data_config.PV_FORECAST = False
-data_config.PV_HISTORY = False
+data_config.PV_FORECAST = True
+data_config.PV_HISTORY = True
 
-data_config.WIND_FORECAST = False
-data_config.WIND_HISTORY = False
+data_config.WIND_FORECAST = True
+data_config.WIND_HISTORY = True
 
-data_config.NOMINAL_NET_POSITION = False
+data_config.NOMINAL_NET_POSITION = True
 
 data_config = task.connect(data_config, name="data_features")
 
@@ -42,7 +42,7 @@ print("Input dim: ", inputDim)
 model_parameters = {
     "epochs": 15000,
     "learning_rate": 0.0001,
-    "hidden_sizes": [256, 256],
+    "hidden_sizes": [512, 512],
     "time_dim": 8,
 }