VictorMylle/Thesis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Plots to compare between quantile regression and diffusion

Browse Source
This commit is contained in:
Victor Mylle
2024-02-18 19:21:59 +01:00
parent bd250a664b
commit 174a82fab2
5 changed files with 123 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/data/dataset.py
View File

@@ -25,12 +25,19 @@ class NrvDataset(Dataset):
self.sequence_length = sequence_length
self.predict_sequence_length = predict_sequence_length
self.samples_to_skip = self.skip_samples(dataframe=dataframe)
self.samples_to_skip = self.skip_samples(dataframe=dataframe, full_day_skip=self.full_day_skip)
total_indices = set(
range(len(dataframe) - self.sequence_length - self.predict_sequence_length)
)
self.valid_indices = sorted(list(total_indices - set(self.samples_to_skip)))
# full day indices
full_day_skipped_samples = self.skip_samples(dataframe=dataframe, full_day_skip=True)
full_day_total_indices = set(
range(len(dataframe) - self.sequence_length - self.predict_sequence_length)
)
self.full_day_valid_indices = sorted(list(full_day_total_indices - set(full_day_skipped_samples)))
self.history_features = []
if self.data_config.LOAD_HISTORY:
self.history_features.append("total_load")
@@ -73,7 +80,7 @@ class NrvDataset(Dataset):
self.history_features, self.forecast_features = self.preprocess_data(dataframe)
def skip_samples(self, dataframe):
def skip_samples(self, dataframe, full_day_skip):
nan_rows = dataframe[dataframe.isnull().any(axis=1)]
nan_indices = nan_rows.index
skip_indices = [
@@ -91,7 +98,7 @@ class NrvDataset(Dataset):
# add indices that are not the start of a day (00:15) to the skip indices (use datetime column)
# get indices of all 00:15 timestamps
if self.full_day_skip:
if full_day_skip:
start_of_day_indices = dataframe[
dataframe["datetime"].dt.time != pd.Timestamp("00:00:00").time()
].index

66
src/trainers/autoregressive_trainer.py
View File

@@ -33,67 +33,29 @@ class AutoRegressiveTrainer(Trainer):
self.model.output_size = 1
def debug_plots(self, task, train: bool, data_loader, sample_indices, epoch):
num_samples = len(sample_indices)
rows = num_samples # One row per sample since we only want one column
# check if self has get_plot_error
if hasattr(self, "get_plot_error"):
cols = 2
print("Using get_plot_error")
else:
cols = 1
print("Using get_plot")
fig = make_subplots(
rows=rows,
cols=cols,
subplot_titles=[f"Sample {i+1}" for i in range(num_samples)],
)
for i, idx in enumerate(sample_indices):
auto_regressive_output = self.auto_regressive(data_loader.dataset, [idx])
for actual_idx, idx in sample_indices.items():
auto_regressive_output = self.auto_regressive(data_loader.dataset, [idx]*1000)
if len(auto_regressive_output) == 3:
initial, predictions, target = auto_regressive_output
else:
initial, predictions, _, target = auto_regressive_output
initial, _, predictions, target = auto_regressive_output
initial = initial.squeeze(0)
predictions = predictions.squeeze(0)
target = target.squeeze(0)
# keep one initial
initial = initial[0]
target = target[0]
sub_fig = self.get_plot(initial, target, predictions, show_legend=(i == 0))
predictions = predictions
row = i + 1
col = 1
fig = self.get_plot(initial, target, predictions, show_legend=(0 == 0))
for trace in sub_fig.data:
fig.add_trace(trace, row=row, col=col)
if cols == 2:
error_sub_fig = self.get_plot_error(
target, predictions
)
for trace in error_sub_fig.data:
fig.add_trace(trace, row=row, col=col + 1)
loss = self.criterion(
predictions.to(self.device), target.to(self.device)
).item()
fig["layout"]["annotations"][i].update(
text=f"{self.criterion.__class__.__name__}: {loss:.6f}"
task.get_logger().report_matplotlib_figure(
title="Training" if train else "Testing",
series=f'Sample {actual_idx}',
iteration=epoch,
figure=fig,
)
# y axis same for all plots
# fig.update_yaxes(range=[-1, 1], col=1)
fig.update_layout(height=1000 * rows)
task.get_logger().report_plotly(
title=f"{'Training' if train else 'Test'} Samples",
series="full_day",
iteration=epoch,
figure=fig,
)
def auto_regressive(self, data_loader, idx, sequence_length: int = 96):
self.model.eval()

17
src/trainers/diffusion_trainer.py
View File

@@ -96,7 +96,16 @@ class DiffusionTrainer:
else:
loader = test_loader
indices = np.random.randint(0, len(loader.dataset) - 1, size=num_samples)
# set seed
np.random.seed(42)
actual_indices = np.random.choice(loader.dataset.full_day_valid_indices, num_samples, replace=False)
indices = {}
for i in actual_indices:
indices[i] = loader.dataset.valid_indices.index(i)
print(actual_indices)
return indices
def init_clearml_task(self, task):
@@ -171,7 +180,7 @@ class DiffusionTrainer:
def debug_plots(self, task, training: bool, data_loader, sample_indices, epoch):
for i, idx in enumerate(sample_indices):
for actual_idx, idx in sample_indices.items():
features, target, _ = data_loader.dataset[idx]
features = features.to(self.device)
@@ -180,6 +189,8 @@ class DiffusionTrainer:
self.model.eval()
with torch.no_grad():
samples = self.sample(self.model, 100, features).cpu().numpy()
samples = self.data_processor.inverse_transform(samples)
target = self.data_processor.inverse_transform(target)
ci_99_upper = np.quantile(samples, 0.995, axis=0)
ci_99_lower = np.quantile(samples, 0.005, axis=0)
@@ -218,7 +229,7 @@ class DiffusionTrainer:
task.get_logger().report_matplotlib_figure(
title="Training" if training else "Testing",
series=f'Sample {i}',
series=f'Sample {actual_idx}',
iteration=epoch,
figure=fig,
)

113
src/trainers/quantile_trainer.py
View File

@@ -10,7 +10,9 @@ import plotly.graph_objects as go
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import CubicSpline
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.patches as mpatches
def sample_from_dist(quantiles, preds):
if isinstance(preds, torch.Tensor):
@@ -261,35 +263,35 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
name="test_CRPS_from_samples_transformed", value=np.mean(crps_from_samples_metric)
)
def get_plot_error(
self,
next_day,
predictions,
):
metric = PinballLoss(quantiles=self.quantiles)
fig = go.Figure()
# def get_plot_error(
# self,
# next_day,
# predictions,
# ):
# metric = PinballLoss(quantiles=self.quantiles)
# fig = go.Figure()
next_day_np = next_day.view(-1).cpu().numpy()
predictions_np = predictions.cpu().numpy()
# next_day_np = next_day.view(-1).cpu().numpy()
# predictions_np = predictions.cpu().numpy()
if True:
next_day_np = self.data_processor.inverse_transform(next_day_np)
predictions_np = self.data_processor.inverse_transform(predictions_np)
# if True:
# next_day_np = self.data_processor.inverse_transform(next_day_np)
# predictions_np = self.data_processor.inverse_transform(predictions_np)
# for each time step, calculate the error using the metric
errors = []
for i in range(96):
# # for each time step, calculate the error using the metric
# errors = []
# for i in range(96):
target_tensor = torch.tensor(next_day_np[i]).unsqueeze(0)
prediction_tensor = torch.tensor(predictions_np[i]).unsqueeze(0)
# target_tensor = torch.tensor(next_day_np[i]).unsqueeze(0)
# prediction_tensor = torch.tensor(predictions_np[i]).unsqueeze(0)
errors.append(metric(prediction_tensor, target_tensor))
# errors.append(metric(prediction_tensor, target_tensor))
# plot the error
fig.add_trace(go.Scatter(x=np.arange(96), y=errors, name=metric.__class__.__name__))
fig.update_layout(title=f"Error of {metric.__class__.__name__} for each time step")
# # plot the error
# fig.add_trace(go.Scatter(x=np.arange(96), y=errors, name=metric.__class__.__name__))
# fig.update_layout(title=f"Error of {metric.__class__.__name__} for each time step")
return fig
# return fig
def get_plot(
@@ -312,26 +314,59 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
next_day_np = self.data_processor.inverse_transform(next_day_np)
predictions_np = self.data_processor.inverse_transform(predictions_np)
ci_99_upper = np.quantile(predictions_np, 0.995, axis=0)
ci_99_lower = np.quantile(predictions_np, 0.005, axis=0)
ci_95_upper = np.quantile(predictions_np, 0.975, axis=0)
ci_95_lower = np.quantile(predictions_np, 0.025, axis=0)
ci_90_upper = np.quantile(predictions_np, 0.95, axis=0)
ci_90_lower = np.quantile(predictions_np, 0.05, axis=0)
ci_50_lower = np.quantile(predictions_np, 0.25, axis=0)
ci_50_upper = np.quantile(predictions_np, 0.75, axis=0)
# Add traces for current and next day
fig.add_trace(go.Scatter(x=np.arange(96), y=current_day_np, name="Current Day"))
fig.add_trace(go.Scatter(x=96 + np.arange(96), y=next_day_np, name="Next Day"))
# fig.add_trace(go.Scatter(x=np.arange(96), y=current_day_np, name="Current Day"))
# fig.add_trace(go.Scatter(x=96 + np.arange(96), y=next_day_np, name="Next Day"))
for i, q in enumerate(self.quantiles):
fig.add_trace(
go.Scatter(
x=96 + np.arange(96),
y=predictions_np[:, i],
name=f"Prediction (Q={q})",
line=dict(dash="dash"),
)
)
# for i, q in enumerate(self.quantiles):
# fig.add_trace(
# go.Scatter(
# x=96 + np.arange(96),
# y=predictions_np[:, i],
# name=f"Prediction (Q={q})",
# line=dict(dash="dash"),
# )
# )
# Update the layout
fig.update_layout(
title="Predictions and Quantiles of the Linear Model",
showlegend=show_legend,
)
# # Update the layout
# fig.update_layout(
# title="Predictions and Quantiles of the Linear Model",
# showlegend=show_legend,
# )
sns.set_theme()
time_steps = np.arange(0, 96)
fig, ax = plt.subplots(figsize=(20, 10))
ax.plot(time_steps, predictions_np.mean(axis=0), label="Mean of NRV samples", linewidth=3)
# ax.fill_between(time_steps, ci_lower, ci_upper, color='b', alpha=0.2, label='Full Interval')
ax.fill_between(time_steps, ci_99_lower, ci_99_upper, color='b', alpha=0.2, label='99% Interval')
ax.fill_between(time_steps, ci_95_lower, ci_95_upper, color='b', alpha=0.2, label='95% Interval')
ax.fill_between(time_steps, ci_90_lower, ci_90_upper, color='b', alpha=0.2, label='90% Interval')
ax.fill_between(time_steps, ci_50_lower, ci_50_upper, color='b', alpha=0.2, label='50% Interval')
ax.plot(next_day_np, label="Real NRV", linewidth=3)
# full_interval_patch = mpatches.Patch(color='b', alpha=0.2, label='Full Interval')
ci_99_patch = mpatches.Patch(color='b', alpha=0.3, label='99% Interval')
ci_95_patch = mpatches.Patch(color='b', alpha=0.4, label='95% Interval')
ci_90_patch = mpatches.Patch(color='b', alpha=0.5, label='90% Interval')
ci_50_patch = mpatches.Patch(color='b', alpha=0.6, label='50% Interval')
ax.legend(handles=[ci_99_patch, ci_95_patch, ci_90_patch, ci_50_patch, ax.lines[0], ax.lines[1]])
return fig
def auto_regressive(self, dataset, idx_batch, sequence_length: int = 96):

15
src/trainers/trainer.py
View File

@@ -86,7 +86,7 @@ class Trainer:
def random_samples(self, train: bool = True, num_samples: int = 10):
train_loader, test_loader = self.data_processor.get_dataloaders(
predict_sequence_length=self.model.output_size
predict_sequence_length=96
)
if train:
@@ -94,7 +94,14 @@ class Trainer:
else:
loader = test_loader
indices = np.random.randint(0, len(loader.dataset) - 1, size=num_samples)
np.random.seed(42)
actual_indices = np.random.choice(loader.dataset.full_day_valid_indices, num_samples, replace=False)
indices = {}
for i in actual_indices:
indices[i] = loader.dataset.valid_indices.index(i)
print(actual_indices)
return indices
def train(self, epochs: int, remotely: bool = False, task: Task = None):
@@ -107,8 +114,8 @@ class Trainer:
predict_sequence_length=self.model.output_size
)
train_samples = self.random_samples(train=True)
test_samples = self.random_samples(train=False)
train_samples = self.random_samples(train=True, num_samples=5)
test_samples = self.random_samples(train=False, num_samples=5)
self.init_clearml_task(task)