diff --git a/src/data/dataset.py b/src/data/dataset.py index 21408f3..bd31402 100644 --- a/src/data/dataset.py +++ b/src/data/dataset.py @@ -25,12 +25,19 @@ class NrvDataset(Dataset): self.sequence_length = sequence_length self.predict_sequence_length = predict_sequence_length - self.samples_to_skip = self.skip_samples(dataframe=dataframe) + self.samples_to_skip = self.skip_samples(dataframe=dataframe, full_day_skip=self.full_day_skip) total_indices = set( range(len(dataframe) - self.sequence_length - self.predict_sequence_length) ) self.valid_indices = sorted(list(total_indices - set(self.samples_to_skip))) + # full day indices + full_day_skipped_samples = self.skip_samples(dataframe=dataframe, full_day_skip=True) + full_day_total_indices = set( + range(len(dataframe) - self.sequence_length - self.predict_sequence_length) + ) + self.full_day_valid_indices = sorted(list(full_day_total_indices - set(full_day_skipped_samples))) + self.history_features = [] if self.data_config.LOAD_HISTORY: self.history_features.append("total_load") @@ -73,7 +80,7 @@ class NrvDataset(Dataset): self.history_features, self.forecast_features = self.preprocess_data(dataframe) - def skip_samples(self, dataframe): + def skip_samples(self, dataframe, full_day_skip): nan_rows = dataframe[dataframe.isnull().any(axis=1)] nan_indices = nan_rows.index skip_indices = [ @@ -91,7 +98,7 @@ class NrvDataset(Dataset): # add indices that are not the start of a day (00:15) to the skip indices (use datetime column) # get indices of all 00:15 timestamps - if self.full_day_skip: + if full_day_skip: start_of_day_indices = dataframe[ dataframe["datetime"].dt.time != pd.Timestamp("00:00:00").time() ].index diff --git a/src/trainers/autoregressive_trainer.py b/src/trainers/autoregressive_trainer.py index d459bd7..37c489f 100644 --- a/src/trainers/autoregressive_trainer.py +++ b/src/trainers/autoregressive_trainer.py @@ -33,67 +33,29 @@ class AutoRegressiveTrainer(Trainer): self.model.output_size = 1 def debug_plots(self, task, train: bool, data_loader, sample_indices, epoch): - num_samples = len(sample_indices) - rows = num_samples # One row per sample since we only want one column - - # check if self has get_plot_error - if hasattr(self, "get_plot_error"): - cols = 2 - print("Using get_plot_error") - else: - cols = 1 - print("Using get_plot") - - fig = make_subplots( - rows=rows, - cols=cols, - subplot_titles=[f"Sample {i+1}" for i in range(num_samples)], - ) - - for i, idx in enumerate(sample_indices): - auto_regressive_output = self.auto_regressive(data_loader.dataset, [idx]) + for actual_idx, idx in sample_indices.items(): + auto_regressive_output = self.auto_regressive(data_loader.dataset, [idx]*1000) if len(auto_regressive_output) == 3: initial, predictions, target = auto_regressive_output else: - initial, predictions, _, target = auto_regressive_output + initial, _, predictions, target = auto_regressive_output - initial = initial.squeeze(0) - predictions = predictions.squeeze(0) - target = target.squeeze(0) + + # keep one initial + initial = initial[0] + target = target[0] - sub_fig = self.get_plot(initial, target, predictions, show_legend=(i == 0)) + predictions = predictions - row = i + 1 - col = 1 + fig = self.get_plot(initial, target, predictions, show_legend=(0 == 0)) - for trace in sub_fig.data: - fig.add_trace(trace, row=row, col=col) - - if cols == 2: - error_sub_fig = self.get_plot_error( - target, predictions - ) - for trace in error_sub_fig.data: - fig.add_trace(trace, row=row, col=col + 1) - - loss = self.criterion( - predictions.to(self.device), target.to(self.device) - ).item() - - fig["layout"]["annotations"][i].update( - text=f"{self.criterion.__class__.__name__}: {loss:.6f}" + task.get_logger().report_matplotlib_figure( + title="Training" if train else "Testing", + series=f'Sample {actual_idx}', + iteration=epoch, + figure=fig, ) - # y axis same for all plots - # fig.update_yaxes(range=[-1, 1], col=1) - - fig.update_layout(height=1000 * rows) - task.get_logger().report_plotly( - title=f"{'Training' if train else 'Test'} Samples", - series="full_day", - iteration=epoch, - figure=fig, - ) def auto_regressive(self, data_loader, idx, sequence_length: int = 96): self.model.eval() diff --git a/src/trainers/diffusion_trainer.py b/src/trainers/diffusion_trainer.py index 6aa0185..b913c7f 100644 --- a/src/trainers/diffusion_trainer.py +++ b/src/trainers/diffusion_trainer.py @@ -96,7 +96,16 @@ class DiffusionTrainer: else: loader = test_loader - indices = np.random.randint(0, len(loader.dataset) - 1, size=num_samples) + # set seed + np.random.seed(42) + + actual_indices = np.random.choice(loader.dataset.full_day_valid_indices, num_samples, replace=False) + indices = {} + for i in actual_indices: + indices[i] = loader.dataset.valid_indices.index(i) + + print(actual_indices) + return indices def init_clearml_task(self, task): @@ -171,7 +180,7 @@ class DiffusionTrainer: def debug_plots(self, task, training: bool, data_loader, sample_indices, epoch): - for i, idx in enumerate(sample_indices): + for actual_idx, idx in sample_indices.items(): features, target, _ = data_loader.dataset[idx] features = features.to(self.device) @@ -180,6 +189,8 @@ class DiffusionTrainer: self.model.eval() with torch.no_grad(): samples = self.sample(self.model, 100, features).cpu().numpy() + samples = self.data_processor.inverse_transform(samples) + target = self.data_processor.inverse_transform(target) ci_99_upper = np.quantile(samples, 0.995, axis=0) ci_99_lower = np.quantile(samples, 0.005, axis=0) @@ -218,7 +229,7 @@ class DiffusionTrainer: task.get_logger().report_matplotlib_figure( title="Training" if training else "Testing", - series=f'Sample {i}', + series=f'Sample {actual_idx}', iteration=epoch, figure=fig, ) diff --git a/src/trainers/quantile_trainer.py b/src/trainers/quantile_trainer.py index c731f63..d1c8c13 100644 --- a/src/trainers/quantile_trainer.py +++ b/src/trainers/quantile_trainer.py @@ -10,7 +10,9 @@ import plotly.graph_objects as go import numpy as np import matplotlib.pyplot as plt from scipy.interpolate import CubicSpline - +import matplotlib.pyplot as plt +import seaborn as sns +import matplotlib.patches as mpatches def sample_from_dist(quantiles, preds): if isinstance(preds, torch.Tensor): @@ -261,35 +263,35 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer): name="test_CRPS_from_samples_transformed", value=np.mean(crps_from_samples_metric) ) - def get_plot_error( - self, - next_day, - predictions, - ): - metric = PinballLoss(quantiles=self.quantiles) - fig = go.Figure() + # def get_plot_error( + # self, + # next_day, + # predictions, + # ): + # metric = PinballLoss(quantiles=self.quantiles) + # fig = go.Figure() - next_day_np = next_day.view(-1).cpu().numpy() - predictions_np = predictions.cpu().numpy() + # next_day_np = next_day.view(-1).cpu().numpy() + # predictions_np = predictions.cpu().numpy() - if True: - next_day_np = self.data_processor.inverse_transform(next_day_np) - predictions_np = self.data_processor.inverse_transform(predictions_np) + # if True: + # next_day_np = self.data_processor.inverse_transform(next_day_np) + # predictions_np = self.data_processor.inverse_transform(predictions_np) - # for each time step, calculate the error using the metric - errors = [] - for i in range(96): + # # for each time step, calculate the error using the metric + # errors = [] + # for i in range(96): - target_tensor = torch.tensor(next_day_np[i]).unsqueeze(0) - prediction_tensor = torch.tensor(predictions_np[i]).unsqueeze(0) + # target_tensor = torch.tensor(next_day_np[i]).unsqueeze(0) + # prediction_tensor = torch.tensor(predictions_np[i]).unsqueeze(0) - errors.append(metric(prediction_tensor, target_tensor)) + # errors.append(metric(prediction_tensor, target_tensor)) - # plot the error - fig.add_trace(go.Scatter(x=np.arange(96), y=errors, name=metric.__class__.__name__)) - fig.update_layout(title=f"Error of {metric.__class__.__name__} for each time step") + # # plot the error + # fig.add_trace(go.Scatter(x=np.arange(96), y=errors, name=metric.__class__.__name__)) + # fig.update_layout(title=f"Error of {metric.__class__.__name__} for each time step") - return fig + # return fig def get_plot( @@ -312,26 +314,59 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer): next_day_np = self.data_processor.inverse_transform(next_day_np) predictions_np = self.data_processor.inverse_transform(predictions_np) + ci_99_upper = np.quantile(predictions_np, 0.995, axis=0) + ci_99_lower = np.quantile(predictions_np, 0.005, axis=0) + + ci_95_upper = np.quantile(predictions_np, 0.975, axis=0) + ci_95_lower = np.quantile(predictions_np, 0.025, axis=0) + + ci_90_upper = np.quantile(predictions_np, 0.95, axis=0) + ci_90_lower = np.quantile(predictions_np, 0.05, axis=0) + + ci_50_lower = np.quantile(predictions_np, 0.25, axis=0) + ci_50_upper = np.quantile(predictions_np, 0.75, axis=0) + # Add traces for current and next day - fig.add_trace(go.Scatter(x=np.arange(96), y=current_day_np, name="Current Day")) - fig.add_trace(go.Scatter(x=96 + np.arange(96), y=next_day_np, name="Next Day")) + # fig.add_trace(go.Scatter(x=np.arange(96), y=current_day_np, name="Current Day")) + # fig.add_trace(go.Scatter(x=96 + np.arange(96), y=next_day_np, name="Next Day")) - for i, q in enumerate(self.quantiles): - fig.add_trace( - go.Scatter( - x=96 + np.arange(96), - y=predictions_np[:, i], - name=f"Prediction (Q={q})", - line=dict(dash="dash"), - ) - ) + # for i, q in enumerate(self.quantiles): + # fig.add_trace( + # go.Scatter( + # x=96 + np.arange(96), + # y=predictions_np[:, i], + # name=f"Prediction (Q={q})", + # line=dict(dash="dash"), + # ) + # ) - # Update the layout - fig.update_layout( - title="Predictions and Quantiles of the Linear Model", - showlegend=show_legend, - ) + # # Update the layout + # fig.update_layout( + # title="Predictions and Quantiles of the Linear Model", + # showlegend=show_legend, + # ) + sns.set_theme() + time_steps = np.arange(0, 96) + + fig, ax = plt.subplots(figsize=(20, 10)) + ax.plot(time_steps, predictions_np.mean(axis=0), label="Mean of NRV samples", linewidth=3) + # ax.fill_between(time_steps, ci_lower, ci_upper, color='b', alpha=0.2, label='Full Interval') + + ax.fill_between(time_steps, ci_99_lower, ci_99_upper, color='b', alpha=0.2, label='99% Interval') + ax.fill_between(time_steps, ci_95_lower, ci_95_upper, color='b', alpha=0.2, label='95% Interval') + ax.fill_between(time_steps, ci_90_lower, ci_90_upper, color='b', alpha=0.2, label='90% Interval') + ax.fill_between(time_steps, ci_50_lower, ci_50_upper, color='b', alpha=0.2, label='50% Interval') + + ax.plot(next_day_np, label="Real NRV", linewidth=3) + # full_interval_patch = mpatches.Patch(color='b', alpha=0.2, label='Full Interval') + ci_99_patch = mpatches.Patch(color='b', alpha=0.3, label='99% Interval') + ci_95_patch = mpatches.Patch(color='b', alpha=0.4, label='95% Interval') + ci_90_patch = mpatches.Patch(color='b', alpha=0.5, label='90% Interval') + ci_50_patch = mpatches.Patch(color='b', alpha=0.6, label='50% Interval') + + + ax.legend(handles=[ci_99_patch, ci_95_patch, ci_90_patch, ci_50_patch, ax.lines[0], ax.lines[1]]) return fig def auto_regressive(self, dataset, idx_batch, sequence_length: int = 96): diff --git a/src/trainers/trainer.py b/src/trainers/trainer.py index f72c4e2..41e5944 100644 --- a/src/trainers/trainer.py +++ b/src/trainers/trainer.py @@ -86,7 +86,7 @@ class Trainer: def random_samples(self, train: bool = True, num_samples: int = 10): train_loader, test_loader = self.data_processor.get_dataloaders( - predict_sequence_length=self.model.output_size + predict_sequence_length=96 ) if train: @@ -94,7 +94,14 @@ class Trainer: else: loader = test_loader - indices = np.random.randint(0, len(loader.dataset) - 1, size=num_samples) + np.random.seed(42) + actual_indices = np.random.choice(loader.dataset.full_day_valid_indices, num_samples, replace=False) + indices = {} + for i in actual_indices: + indices[i] = loader.dataset.valid_indices.index(i) + + print(actual_indices) + return indices def train(self, epochs: int, remotely: bool = False, task: Task = None): @@ -107,8 +114,8 @@ class Trainer: predict_sequence_length=self.model.output_size ) - train_samples = self.random_samples(train=True) - test_samples = self.random_samples(train=False) + train_samples = self.random_samples(train=True, num_samples=5) + test_samples = self.random_samples(train=False, num_samples=5) self.init_clearml_task(task)