Implemented imbalance price reconstruction

Result-Reports/December.md

@@ -2,7 +2,7 @@
 - [ ] Quantiles zelf breder maken na fitten, literatuur bekijken (overconfident voor ondergrens) (Validation set januari 2023)
 Eerst literatuur bekijken ofdat probleem al voorkomt
 
-- [ ] compare reconstructed prices with real imbalance prices (on figure and metrics on whole test set)
+- [x] compare reconstructed prices with real imbalance prices (on figure and metrics on whole test set)
 
 - [ ] Baseline policy (Eche imbalanceprijzen):
   - Batterij:

Result-Reports/Imbalance_price_reconstruction.md

@@ -0,0 +1,27 @@
+# Imbalance Price Reconstruction
+Some things to note: \
+SI = ACE - NRV but we only know NRV. We take the SI as the -NRV.\
+There is also some uncertainty on how the bid ladder is constructed and what factors are taken into account to calculate the imbalance price. We will try some different methods to reconstruct the imbalance price and see which one is the best.\
+
+The following examples were reconstructed by only using the real NRV values and the bids published by Elia.
+
+Method 1: Sort bids on price
+Method 2: Sort bids on activation order and then on price (Elia says they use this method)
+
+![](imbalance_prices_images/imbalance_price_reconstruction_04-07-2023.png)
+![](imbalance_prices_images/imbalance_price_reconstruction_04-11-2023.png)
+![](imbalance_prices_images/imbalance_price_reconstruction_04-12-2023.png)
+![](imbalance_prices_images/imbalance_price_reconstruction_06-08-2023.png)
+![](imbalance_prices_images/imbalance_price_reconstruction_14-02-2023.png)
+![](imbalance_prices_images/imbalance_price_reconstruction_14-03-2023.png)
+![](imbalance_prices_images/imbalance_price_reconstruction_16-02-2023.png)
+![](imbalance_prices_images/imbalance_price_reconstruction_17-08-2023.png)
+![](imbalance_prices_images/imbalance_price_reconstruction_29-08-2023.png)
+
+From these examples we can see that just sorting based on price is given better reconstructions than the method used by Elia???????? This is something weird.
+
+# Mean Absolute Error
+| Method | MAE |
+| --- | --- |
+| Method 1 | 1842235.14 |
+| Method 2 | 4461725.94 |

src/utils/imbalance_price_calculator.py

@@ -1,71 +1,100 @@
 from datetime import datetime
 import plotly.graph_objects as go
 import numpy as np
+import pandas as pd
 
+incremental_bids = "../../data/incremental_bids.csv"
+decremental_bids = "../../data/decremental_bids.csv"
 
 class ImbalancePriceCalculator:
-    def __init__(self, bids) -> None:
-        self.bids = bids
-        self.bid_ladder = self.generate_bid_ladder()
+    def __init__(self, method: int = 1) -> None:
+        self.method = method
+        self.load_bids()
 
-    def generate_bid_ladder(self):
-        # Sort bids by Product (3th column of tuple, use order defined above) and then by price (second column of tuple)
-        # first sort on activaation order, then on bid price
-        bids = sorted(self.bids, key=lambda x: (x[3], x[1]))
+    def load_bids(self):
+        df = pd.read_csv(incremental_bids, sep=";")
+        df["Datetime"] = pd.to_datetime(df["Datetime"], utc=True)
     
-        # Calculate cummulative bids
-        cummulative_volume = 0
-        cummulative_bids = []
-        max_price = 0
-        for volume, price, _, order in bids:
-            cummulative_volume += volume
-            if price > max_price:
-                max_price = price
-            cummulative_bids.append((cummulative_volume, max_price, price, order))
-            print((cummulative_volume, max_price, price, order))
+        # sort by Datetime, Activation Order, Bid Price
+        df = df.sort_values(by=["Datetime", "Activation Order", "Bid Price"])
 
-        cummulative_bids = [(vol, price) for vol, price, _, _ in cummulative_bids]
+        # next we need to calculate the cummulative bids for every datetime 
+        incremental_cum_df = df.groupby(["Datetime"]).apply(self.generate_bid_ladder)
 
-        return cummulative_bids
+        df = pd.read_csv(decremental_bids, sep=";")
+        df["Datetime"] = pd.to_datetime(df["Datetime"], utc=True)
+        decremental_cum_df = df.groupby(["Datetime"]).apply(self.generate_bid_ladder, incremental=False)
 
-    def get_imbalance_price(self, volume):
-        for bid in self.bid_ladder:
+        # merge the two dataframes on the Datetime
+        self.bid_ladder = pd.merge(incremental_cum_df, decremental_cum_df, on="Datetime", how="inner", suffixes=("_inc", "_dec"))
+        self.bid_ladder = self.bid_ladder.sort_values(by=["Datetime"])
+    
+    def generate_bid_ladder(self, bids, incremental: bool = True):
+        if self.method == 1:
+            bids_df = bids.sort_values(by=["Bid Price"], ascending=[incremental])
+        elif self.method == 2:
+            bids_df = bids.sort_values(by=["Activation Order", "Bid Price"], ascending=[True, incremental])
+        bids_df['cumulative_volume'] = bids_df['Bid Volume'].cumsum()
+        if incremental:
+            bids_df['max_price'] = bids_df['Bid Price'].cummax()
+        else:
+            bids_df['max_price'] = bids_df['Bid Price'].cummin()
+        bid_ladder = list(zip(bids_df['cumulative_volume'], bids_df['max_price']))
+        bid_ladder_df = pd.DataFrame([[bid_ladder]], columns=['bid_ladder'])
+
+        return bid_ladder_df
+    
+    def check_datetime(self, datetime):
+        return datetime in self.bid_ladder.index
+           
+    
+    def get_imbalance_price(self, datetime, volume):
+        if volume < 0:
+            volume = -volume
+            bid_ladder = self.bid_ladder.loc[self.bid_ladder.index == datetime]["bid_ladder_dec"].values[0]
+        else:
+            bid_ladder = self.bid_ladder.loc[self.bid_ladder.index == datetime]["bid_ladder_inc"].values[0]
+
+        for bid in bid_ladder:
             if bid[0] > volume:
                 return bid[1]
-        return self.bid_ladder[-1][1]
+        return bid_ladder[-1][1]
+
+
+    def plot(self, datetime):
+        row = self.bid_ladder.loc[self.bid_ladder.index == datetime]
+        dec_bids = row["bid_ladder_dec"].values[0]
+        inc_bids = row["bid_ladder_inc"].values[0]
+        
+
 
-    def plot(self):
         # Prepare data for plot
-        x_interpolated = [vol for i in range(len(self.bid_ladder) - 1) for vol in [self.bid_ladder[i][0], self.bid_ladder[i+1][0]]]
-        y_interpolated = [price for cum_vol, price in self.bid_ladder for _ in (0, 1)]
+        x_inc_interpolated = [vol for i in range(len(inc_bids) - 1) for vol in [inc_bids[i][0], inc_bids[i+1][0]]]
+        y_inc_interpolated = [price for cum_vol, price in inc_bids for _ in (0, 1)]
+
+        x_dec_interpolated = [-vol for i in range(len(dec_bids) - 1) for vol in [dec_bids[i][0], dec_bids[i+1][0]]]
+        y_dec_interpolated = [price for cum_vol, price in dec_bids for _ in (0, 1)]
 
         # Create and show the plot make sure hovering works in between the steps
         fig = go.Figure()
-        fig.add_trace(go.Scatter(x=x_interpolated, y=y_interpolated, mode='lines+markers'))
+        fig.add_trace(go.Scatter(x=x_inc_interpolated, y=y_inc_interpolated, mode='lines+markers', name="inc"))
+
+
+        fig.add_trace(go.Scatter(x=x_dec_interpolated, y=y_dec_interpolated, mode='lines+markers', name="dec"))
         fig.update_layout(
             title='Bid ladder',
             xaxis_title='Volume',
             yaxis_title='Price',
             hovermode='x unified'
         )
+
         fig.show()
 
-        # also print the prices needed for 100MW, 200MW, 300MW, 400MW, 500MW, 600MW, 700MW, 800MW, 900MW, 1000MW
-        for i in range(1, 11):
-            # get first cummulative_bid i where i*100 > cummulative_volume of i and smaller than i+1
-            
-            mw = 100
-            
-            for bid in self.bid_ladder:
-                if bid[0] > mw*i:
-                    print(f"{mw*i}MW: {bid[1]}")
-                    break
-
-        print(f"Max: {self.bid_ladder[-1][1]}")
-
-    def get_imbalance_price_2023(self, NRV_PREV, NRV):
-        pass
+    def get_imbalance_price_2023(self, datetime, NRV_PREV, NRV):
+        MIP = self.get_imbalance_price(datetime, abs(NRV))
+        MDP = self.get_imbalance_price(datetime, -abs(NRV))
 
+        return calculate_imbalance_price_2023(-NRV_PREV, -NRV, MIP, MDP)
 
 # SI = -NRV
 def calculate_imbalance_price_2023(SI_PREV: float, SI: float, MIP: float, MDP: float):
@@ -108,7 +137,3 @@ def calculate_imbalance_price_2023(SI_PREV: float, SI: float, MIP: float, MDP: f
 
     # return alpha, imbalance price
     return alpha, imbalance_price
-
-
-
-