VictorMylle/Thesis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Sped up sampling 20x

Browse Source
This commit is contained in:
Victor Mylle
2023-11-25 18:09:42 +00:00
parent 5de3f64a1a
commit 300f268286
10 changed files with 498 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
src/notebooks/training.ipynb
View File

@@ -33,15 +33,15 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"#### Data Processor ####\n",
"data_config = DataConfig()\n",
"data_config.NRV_HISTORY = True\n",
"data_config.LOAD_HISTORY = True\n",
"data_config.LOAD_FORECAST = True\n",
"data_config.LOAD_HISTORY = False\n",
"data_config.LOAD_FORECAST = False\n",
"\n",
"data_config.WIND_FORECAST = False\n",
"data_config.WIND_HISTORY = False\n",
@@ -60,35 +60,33 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Can't get url information for git repo in /workspaces/Thesis/src/notebooks\n"
"InsecureRequestWarning: Certificate verification is disabled! Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"ClearML Task: created new task id=599152a9e44d4ba6a5bcb603e5041b01\n",
"ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/599152a9e44d4ba6a5bcb603e5041b01/output/log\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"JSON serialization of artifact 'dictionary' failed, reverting to pickle\n"
"ClearML Task: created new task id=8423d146953041eba8d7b4c27d7ed6a5\n",
"ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/8423d146953041eba8d7b4c27d7ed6a5/output/log\n",
"2023-11-23 23:07:35,461 - clearml.Task - INFO - Storing jupyter notebook directly as code\n",
"2023-11-23 23:07:39,250 - clearml - WARNING - JSON serialization of artifact 'dictionary' failed, reverting to pickle\n"
]
}
],
"source": [
"data_processor.set_full_day_skip(True)\n",
"quantiles = [0.01, 0.05, 0.1, 0.15, 0.4, 0.5, 0.6, 0.85, 0.9, 0.95, 0.99]\n",
"trainer = ProbabilisticBaselineTrainer(quantiles=quantiles, data_processor=data_processor, clearml_helper=clearml_helper)\n",
"trainer = ProbabilisticBaselineTrainer(\n",
" quantiles=quantiles, data_processor=data_processor, clearml_helper=clearml_helper\n",
")\n",
"trainer.add_metrics_to_track([CRPSLoss(quantiles=quantiles)])\n",
"trainer.train()"
]
@@ -160,9 +158,32 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {},
"outputs": [],
"outputs": [
{
"ename": "ParserError",
"evalue": "Error tokenizing data. C error: Calling read(nbytes) on source failed. Try engine='python'.",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mParserError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/workspaces/Thesis/src/notebooks/training.ipynb Cell 8\u001b[0m line \u001b[0;36m1\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#X10sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m data_config \u001b[39m=\u001b[39m DataConfig()\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#X10sdnNjb2RlLXJlbW90ZQ%3D%3D?line=11'>12</a>\u001b[0m data_config\u001b[39m.\u001b[39mLOAD_FORECAST \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#X10sdnNjb2RlLXJlbW90ZQ%3D%3D?line=12'>13</a>\u001b[0m data_processor \u001b[39m=\u001b[39m DataProcessor(data_config)\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#X10sdnNjb2RlLXJlbW90ZQ%3D%3D?line=13'>14</a>\u001b[0m data_processor\u001b[39m.\u001b[39mset_batch_size(\u001b[39m1024\u001b[39m)\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#X10sdnNjb2RlLXJlbW90ZQ%3D%3D?line=16'>17</a>\u001b[0m data_processor\u001b[39m.\u001b[39mset_train_range((datetime(year\u001b[39m=\u001b[39m\u001b[39m2015\u001b[39m, month\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m, day\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m, tzinfo\u001b[39m=\u001b[39mpytz\u001b[39m.\u001b[39mUTC), datetime(year\u001b[39m=\u001b[39m\u001b[39m2022\u001b[39m, month\u001b[39m=\u001b[39m\u001b[39m11\u001b[39m, day\u001b[39m=\u001b[39m\u001b[39m30\u001b[39m, tzinfo\u001b[39m=\u001b[39mpytz\u001b[39m.\u001b[39mUTC)))\n",
"File \u001b[0;32m/workspaces/Thesis/src/notebooks/../data/preprocessing.py:52\u001b[0m, in \u001b[0;36mDataProcessor.__init__\u001b[0;34m(self, data_config)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhistory_features \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_nrv_history()\n\u001b[1;32m 51\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfuture_features \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_load_forecast()\n\u001b[0;32m---> 52\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpv_forecast \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mget_pv_forecast()\n\u001b[1;32m 53\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mwind_forecast \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_wind_forecast()\n\u001b[1;32m 55\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mall_features \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhistory_features\u001b[39m.\u001b[39mmerge(\n\u001b[1;32m 56\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfuture_features, on\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mdatetime\u001b[39m\u001b[39m\"\u001b[39m, how\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mleft\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 57\u001b[0m )\n",
"File \u001b[0;32m/workspaces/Thesis/src/notebooks/../data/preprocessing.py:132\u001b[0m, in \u001b[0;36mDataProcessor.get_pv_forecast\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_pv_forecast\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 132\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39;49mread_csv(pv_forecast_data_path, delimiter\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m;\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n\u001b[1;32m 134\u001b[0m df \u001b[39m=\u001b[39m df\u001b[39m.\u001b[39mrename(\n\u001b[1;32m 135\u001b[0m columns\u001b[39m=\u001b[39m{\u001b[39m\"\u001b[39m\u001b[39mdayahead11hforecast\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mpv_forecast\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mDatetime\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mdatetime\u001b[39m\u001b[39m\"\u001b[39m}\n\u001b[1;32m 136\u001b[0m )\n\u001b[1;32m 137\u001b[0m df \u001b[39m=\u001b[39m df[[\u001b[39m\"\u001b[39m\u001b[39mdatetime\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mpv_forecast\u001b[39m\u001b[39m\"\u001b[39m]]\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/io/parsers/readers.py:912\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m 899\u001b[0m kwds_defaults \u001b[39m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m 900\u001b[0m dialect,\n\u001b[1;32m 901\u001b[0m delimiter,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 908\u001b[0m dtype_backend\u001b[39m=\u001b[39mdtype_backend,\n\u001b[1;32m 909\u001b[0m )\n\u001b[1;32m 910\u001b[0m kwds\u001b[39m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m--> 912\u001b[0m \u001b[39mreturn\u001b[39;00m _read(filepath_or_buffer, kwds)\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/io/parsers/readers.py:583\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 580\u001b[0m \u001b[39mreturn\u001b[39;00m parser\n\u001b[1;32m 582\u001b[0m \u001b[39mwith\u001b[39;00m parser:\n\u001b[0;32m--> 583\u001b[0m \u001b[39mreturn\u001b[39;00m parser\u001b[39m.\u001b[39;49mread(nrows)\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1704\u001b[0m, in \u001b[0;36mTextFileReader.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 1697\u001b[0m nrows \u001b[39m=\u001b[39m validate_integer(\u001b[39m\"\u001b[39m\u001b[39mnrows\u001b[39m\u001b[39m\"\u001b[39m, nrows)\n\u001b[1;32m 1698\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 1699\u001b[0m \u001b[39m# error: \"ParserBase\" has no attribute \"read\"\u001b[39;00m\n\u001b[1;32m 1700\u001b[0m (\n\u001b[1;32m 1701\u001b[0m index,\n\u001b[1;32m 1702\u001b[0m columns,\n\u001b[1;32m 1703\u001b[0m col_dict,\n\u001b[0;32m-> 1704\u001b[0m ) \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_engine\u001b[39m.\u001b[39;49mread( \u001b[39m# type: ignore[attr-defined]\u001b[39;49;00m\n\u001b[1;32m 1705\u001b[0m nrows\n\u001b[1;32m 1706\u001b[0m )\n\u001b[1;32m 1707\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 1708\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mclose()\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/io/parsers/c_parser_wrapper.py:234\u001b[0m, in \u001b[0;36mCParserWrapper.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 233\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlow_memory:\n\u001b[0;32m--> 234\u001b[0m chunks \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_reader\u001b[39m.\u001b[39;49mread_low_memory(nrows)\n\u001b[1;32m 235\u001b[0m \u001b[39m# destructive to chunks\u001b[39;00m\n\u001b[1;32m 236\u001b[0m data \u001b[39m=\u001b[39m _concatenate_chunks(chunks)\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/_libs/parsers.pyx:814\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader.read_low_memory\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/_libs/parsers.pyx:875\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/_libs/parsers.pyx:850\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._tokenize_rows\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/_libs/parsers.pyx:861\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._check_tokenize_status\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pandas/_libs/parsers.pyx:2029\u001b[0m, in \u001b[0;36mpandas._libs.parsers.raise_parser_error\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mParserError\u001b[0m: Error tokenizing data. C error: Calling read(nbytes) on source failed. Try engine='python'."
]
}
],
"source": [
"#### Hyperparameters ####\n",
"inputDim = 96\n",
@@ -203,18 +224,18 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/workspaces/Thesis/src/notebooks/../trainers/quantile_trainer.py:27: UserWarning:\n",
"/workspaces/Thesis/src/notebooks/../trainers/quantile_trainer.py:70: UserWarning:\n",
"\n",
"To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
"\n",
"/workspaces/Thesis/src/notebooks/../losses/pinball_loss.py:7: UserWarning:\n",
"/workspaces/Thesis/src/notebooks/../losses/pinball_loss.py:8: UserWarning:\n",
"\n",
"To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
"\n"
@@ -224,30 +245,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
"ClearML Task: created new task id=b2fd376e79b14ba4b26b0436cb130cfe\n",
"ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/b2fd376e79b14ba4b26b0436cb130cfe/output/log\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Can't get url information for git repo in /workspaces/Thesis/src/notebooks\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"ClearML Task: created new task id=215dd7634cf2475693ea6081e2ab7559\n",
"ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/215dd7634cf2475693ea6081e2ab7559/output/log\n",
"Early stopping triggered\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 25804/25804 [20:36<00:00, 20.87it/s]\n"
]
}
],
"source": [
@@ -259,9 +260,10 @@
"\n",
"# quantiles = torch.tensor([0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99]).to(\"cuda\")\n",
"quantiles = torch.tensor(\n",
" [0.01, 0.05, 0.005, 0.1, 0.15, 0.4, 0.5, 0.6, 0.85, 0.9, 0.95, 0.99]\n",
" [0.01, 0.05, 0.1, 0.15, 0.3, 0.4, 0.5, 0.6, 0.7, 0.85, 0.9, 0.95, 0.99]\n",
").to(\"cuda\")\n",
"\n",
"# model = LinearRegression(inputDim, len(quantiles))\n",
"model = NonLinearRegression(inputDim, len(quantiles), hiddenSize=1024, numLayers=5)\n",
"optimizer = torch.optim.Adam(model.parameters(), lr=learningRate)\n",
"\n",
@@ -292,18 +294,18 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/workspaces/Thesis/src/notebooks/../trainers/quantile_trainer.py:290: UserWarning:\n",
"/workspaces/Thesis/src/notebooks/../trainers/quantile_trainer.py:335: UserWarning:\n",
"\n",
"To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
"\n",
"/workspaces/Thesis/src/notebooks/../losses/pinball_loss.py:21: UserWarning:\n",
"/workspaces/Thesis/src/notebooks/../losses/pinball_loss.py:23: UserWarning:\n",
"\n",
"To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
"\n"
@@ -313,21 +315,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"ClearML Task: created new task id=c50a62963dd649c387f1122ccee61d2f\n",
"ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/c50a62963dd649c387f1122ccee61d2f/output/log\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Can't get url information for git repo in /workspaces/Thesis/src/notebooks\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"ClearML Task: created new task id=160b4938ae3145db9ef8b55e71452987\n",
"ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/160b4938ae3145db9ef8b55e71452987/output/log\n",
"Early stopping triggered\n"
]
},
@@ -335,7 +324,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"/workspaces/Thesis/src/notebooks/../trainers/quantile_trainer.py:338: UserWarning:\n",
"/workspaces/Thesis/src/notebooks/../trainers/quantile_trainer.py:366: UserWarning:\n",
"\n",
"Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at /opt/conda/conda-bld/pytorch_1682343967769/work/torch/csrc/utils/tensor_new.cpp:245.)\n",
"\n"
@@ -377,12 +366,111 @@
"trainer.train(epochs=epochs)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"torch.Size([3, 192])\n",
"torch.Size([3, 96])\n"
]
}
],
"source": [
"\n",
"\n",
"dataset = data_processor.get_train_dataloader().dataset\n",
"dataset.predict_sequence_length = 1\n",
"dataset.data_config.LOAD_HISTORY = True\n",
"\n",
"\n",
"def auto_regressive_batch(dataset, idx_batch, sequence_length):\n",
" target_full = [] # (batch_size, sequence_length)\n",
" predictions_samples = [] # (batch_size, sequence_length)\n",
" predictions_full = [] # (batch_size, sequence_length, quantiles)\n",
"\n",
" prev_features, targets = dataset.get_batch(idx_batch)\n",
"\n",
" initial_sequence = prev_features[:, :96]\n",
"\n",
" target_full = targets[:, 0]\n",
" self.\n",
"\n",
"\n",
"\n",
"auto_regressive_batch(dataset, [0, 1, 2], 50)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
"source": [
"def auto_regressive(self, data_loader, idx, sequence_length: int = 96):\n",
" self.model.eval()\n",
" target_full = []\n",
" predictions_sampled = []\n",
" predictions_full = []\n",
"\n",
" prev_features, target = data_loader.dataset[idx]\n",
" prev_features = prev_features.to(self.device)\n",
"\n",
" initial_sequence = prev_features[:96]\n",
"\n",
" target_full.append(target)\n",
" with torch.no_grad():\n",
" prediction = self.model(prev_features.unsqueeze(0))\n",
" predictions_full.append(prediction.squeeze(0))\n",
"\n",
" # sample from the distribution\n",
" sample = self.sample_from_dist(\n",
" self.quantiles.cpu(), prediction.squeeze(-1).cpu().numpy()\n",
" )\n",
" predictions_sampled.append(sample)\n",
"\n",
" for i in range(sequence_length - 1):\n",
" new_features = torch.cat(\n",
" (prev_features[1:96].cpu(), torch.tensor([predictions_sampled[-1]])),\n",
" dim=0,\n",
" )\n",
" new_features = new_features.float()\n",
"\n",
" # get the other needed features\n",
" other_features, new_target = data_loader.dataset.random_day_autoregressive(\n",
" idx + i + 1\n",
" )\n",
"\n",
" if other_features is not None:\n",
" prev_features = torch.cat((new_features, other_features), dim=0)\n",
" else:\n",
" prev_features = new_features\n",
"\n",
" # add target to target_full\n",
" target_full.append(new_target)\n",
"\n",
" # predict\n",
" with torch.no_grad():\n",
" prediction = self.model(prev_features.unsqueeze(0).to(self.device))\n",
" predictions_full.append(prediction.squeeze(0))\n",
"\n",
" # sample from the distribution\n",
" sample = self.sample_from_dist(\n",
" self.quantiles.cpu(), prediction.squeeze(-1).cpu().numpy()\n",
" )\n",
" predictions_sampled.append(sample)\n",
"\n",
" return (\n",
" initial_sequence.cpu(),\n",
" torch.stack(predictions_full).cpu(),\n",
" torch.tensor(predictions_sampled).reshape(-1, 1),\n",
" torch.stack(target_full).cpu(),\n",
" )"
]
}
],
"metadata": {