Initial Commit

src/data/__init__.py

@@ -0,0 +1,2 @@
+from .dataset import NrvDataset
+from .preprocessing import DataProcessor

src/data/dataset.py

@@ -0,0 +1,15 @@
+import torch
+from torch.utils.data import Dataset, DataLoader
+import pandas as pd
+
+class NrvDataset(Dataset):
+    def __init__(self, dataframe, sequence_length=96, predict_sequence_length=96):
+        self.data = torch.tensor(dataframe['nrv'].to_numpy(), dtype=torch.float32)
+        self.sequence_length = sequence_length
+        self.predict_sequence_length = predict_sequence_length
+
+    def __len__(self):
+        return len(self.data) - self.sequence_length - self.predict_sequence_length
+    
+    def __getitem__(self, idx):
+        return self.data[idx:idx+self.sequence_length], self.data[idx+self.sequence_length:idx+self.sequence_length+self.predict_sequence_length]

src/data/preprocessing.py

@@ -0,0 +1,116 @@
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import MinMaxScaler
+import torch
+from data.dataset import NrvDataset
+from datetime import datetime
+import pytz
+
+
+history_data_path = "../../data/history-quarter-hour-data.csv"
+
+class DataProcessor:
+    def __init__(self):
+        self.batch_size = 2048
+
+        self.train_range = (-np.inf, datetime(year=2022, month=11, day=30, tzinfo=pytz.UTC))
+        self.test_range = (datetime(year=2023, month=1, day=1, tzinfo=pytz.UTC), np.inf)
+
+        self.update_range_str()
+
+        self.features = ['nrv']
+
+        self.nrv_df = self.get_nrv_history()
+        self.nrv_scaler = MinMaxScaler(feature_range=(-1, 1))
+
+
+    def set_train_range(self, train_range: tuple):
+        self.train_range = train_range
+        self.update_range_str()
+
+    def set_test_range(self, test_range: tuple):
+        self.test_range = test_range
+        self.update_range_str()
+
+    def update_range_str(self):
+        self.train_range_start = str(self.train_range[0]) if self.train_range[0] != -np.inf else "-inf"
+        self.train_range_end = str(self.train_range[1]) if self.train_range[1] != np.inf else "inf"
+        self.test_range_start = str(self.test_range[0]) if self.test_range[0] != -np.inf else "-inf"
+        self.test_range_end = str(self.test_range[1]) if self.test_range[1] != np.inf else "inf"
+
+    def get_nrv_history(self):
+        df = pd.read_csv(history_data_path, delimiter=';')
+        df = df[['datetime', 'netregulationvolume']]
+        df = df.rename(columns={'netregulationvolume': 'nrv'})
+        df['datetime'] = pd.to_datetime(df['datetime'])
+        counts = df['datetime'].dt.date.value_counts().sort_index()
+        df = df[df['datetime'].dt.date.isin(counts[counts == 96].index)]
+
+        df.sort_values(by="datetime", inplace=True)
+        return df
+    
+    def set_batch_size(self, batch_size: int):
+        self.batch_size = batch_size
+
+    def get_dataloader(self, dataset, shuffle: bool = True):
+        return torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, shuffle=shuffle, num_workers=4)
+
+    def get_train_dataloader(self, transform: bool = True, predict_sequence_length: int = 96):
+        train_df = self.nrv_df.copy()
+
+        if self.train_range[0] != -np.inf:
+            train_df = train_df[(train_df['datetime'] >= self.train_range[0])]
+        
+        if  self.train_range[1] != np.inf:
+            train_df = train_df[(train_df['datetime'] <= self.train_range[1])]
+
+        if transform:
+            train_df['nrv'] = self.nrv_scaler.fit_transform(train_df['nrv'].values.reshape(-1, 1)).reshape(-1)
+        
+        train_dataset = NrvDataset(train_df, predict_sequence_length=predict_sequence_length)
+        return self.get_dataloader(train_dataset)
+
+    def get_test_dataloader(self, transform: bool = True, predict_sequence_length: int = 96):
+
+        test_df = self.nrv_df.copy()
+
+        if self.test_range[0] != -np.inf:
+            test_df = test_df[(test_df['datetime'] >= self.test_range[0])]
+        
+        if  self.test_range[1] != np.inf:
+            test_df = test_df[(test_df['datetime'] <= self.test_range[1])]
+
+        if transform:
+            test_df['nrv'] = self.nrv_scaler.transform(test_df['nrv'].values.reshape(-1, 1)).reshape(-1)
+        test_dataset = NrvDataset(test_df, predict_sequence_length=predict_sequence_length)
+        return self.get_dataloader(test_dataset, shuffle=False)
+    
+    
+    def get_dataloaders(self, transform: bool = True, predict_sequence_length: int = 96):
+        return self.get_train_dataloader(transform=transform, predict_sequence_length=predict_sequence_length), self.get_test_dataloader(transform=transform, predict_sequence_length=predict_sequence_length)
+
+    def get_random_day(self, train: bool = True, transform: bool = True):
+        df = self.nrv_df.copy()
+
+        range = self.train_range if train else self.test_range
+
+        if range[0] != -np.inf:
+            df = df[(df['datetime'] >= range[0])]
+        
+        if  range[1] != np.inf:
+            df = df[(df['datetime'] <= range[1])]
+
+        if transform:
+            df['nrv'] = self.nrv_scaler.transform(df['nrv'].values.reshape(-1, 1)).reshape(-1)
+
+        data_tensor = torch.tensor(df[self.features].values, dtype=torch.float32)
+
+        random_start_idx = np.random.randint(0, len(df) - 191)
+
+        current_day_features = data_tensor[random_start_idx:random_start_idx+96]
+        next_day_features = data_tensor[random_start_idx+96:random_start_idx+192]
+
+        return (current_day_features, next_day_features)
+    
+    def inverse_transform(self, tensor: torch.Tensor):
+        return self.nrv_scaler.inverse_transform(tensor.cpu().numpy()).reshape(-1)