From 3124b54995fc222455a63a7390001e39502cefa5 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 12:52:06 +0000 Subject: [PATCH 01/19] use pad nwp data to make same size as nwp --- .../models/perceiver/perceiver_conv3d_nwp_sat.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index 8f77fbb..4f8c93c 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -141,7 +141,7 @@ def forward(self, x): # ******************* Satellite imagery ************************* # Shape: batch_size, channel, seq_length, height, width # TODO: Use optical flow, not actual sat images of the future! - sat_data = x.satellite.data[0 : self.batch_size].float() + sat_data = x.hrvsatellite.data[0 : self.batch_size].float() if not self.use_future_satellite_images: sat_data[:, -self.forecast_len_5: ] = 0 # This might not be the best way to do it @@ -167,6 +167,15 @@ def forward(self, x): nwp_data_zeros = torch.zeros(size=(batch_size, seq_len - nwp_seq_len, nwp_width, nwp_height, n_nwp_chans), device=nwp_data.device) nwp_data = torch.cat([nwp_data, nwp_data_zeros], dim=1) + # v15 the width and height are a lot less, so lets expand them + nwp_data_zeros = torch.zeros(size=(batch_size, seq_len, width - nwp_width, nwp_height, n_nwp_chans), + device=nwp_data.device) + nwp_data = torch.cat([nwp_data, nwp_data_zeros], dim=1) + nwp_data_zeros = torch.zeros(size=(batch_size, seq_len, width, height - nwp_height, n_nwp_chans), + device=nwp_data.device) + nwp_data = torch.cat([nwp_data, nwp_data_zeros], dim=1) + + nwp_data = nwp_data.reshape(new_batch_size, nwp_width, nwp_height, n_nwp_chans) assert nwp_width == width, f'widths should be the same({nwp_width},{width})' From 5921df1856855ed0f813daaa711c0c5379518506 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 12:52:49 +0000 Subject: [PATCH 02/19] ue satellite data --- predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index 4f8c93c..d9be4a5 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -141,7 +141,7 @@ def forward(self, x): # ******************* Satellite imagery ************************* # Shape: batch_size, channel, seq_length, height, width # TODO: Use optical flow, not actual sat images of the future! - sat_data = x.hrvsatellite.data[0 : self.batch_size].float() + sat_data = x.satellite.data[0 : self.batch_size].float() if not self.use_future_satellite_images: sat_data[:, -self.forecast_len_5: ] = 0 # This might not be the best way to do it From e8cbb577677878fd6f69174d5cf6795e8fdc76f5 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 12:55:56 +0000 Subject: [PATCH 03/19] expand sat data --- .../perceiver/perceiver_conv3d_nwp_sat.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index d9be4a5..c287d7c 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -167,17 +167,16 @@ def forward(self, x): nwp_data_zeros = torch.zeros(size=(batch_size, seq_len - nwp_seq_len, nwp_width, nwp_height, n_nwp_chans), device=nwp_data.device) nwp_data = torch.cat([nwp_data, nwp_data_zeros], dim=1) - # v15 the width and height are a lot less, so lets expand them - nwp_data_zeros = torch.zeros(size=(batch_size, seq_len, width - nwp_width, nwp_height, n_nwp_chans), - device=nwp_data.device) - nwp_data = torch.cat([nwp_data, nwp_data_zeros], dim=1) - nwp_data_zeros = torch.zeros(size=(batch_size, seq_len, width, height - nwp_height, n_nwp_chans), - device=nwp_data.device) - nwp_data = torch.cat([nwp_data, nwp_data_zeros], dim=1) - - nwp_data = nwp_data.reshape(new_batch_size, nwp_width, nwp_height, n_nwp_chans) + # v15 the width and height are a lot less, so lets expand the sat data. There should be a better way + sat_data_zeros = torch.zeros(size=(new_batch_size, nwp_width - width, height, n_chans), + device=sat_data.device) + sat_data = torch.cat([sat_data, sat_data_zeros], dim=1) + sat_data_zeros = torch.zeros(size=(new_batch_size, nwp_width, nwp_height - height, n_chans), + device=sat_data.device) + sat_data = torch.cat([sat_data, sat_data_zeros], dim=1) + assert nwp_width == width, f'widths should be the same({nwp_width},{width})' assert nwp_height == height, f'heights should be the same({nwp_height},{height})' From 569d9fd33ae1bd5a14ceced7eebcdc06ab64dd63 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 12:56:59 +0000 Subject: [PATCH 04/19] update --- predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index c287d7c..e8d59f3 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -175,7 +175,7 @@ def forward(self, x): sat_data = torch.cat([sat_data, sat_data_zeros], dim=1) sat_data_zeros = torch.zeros(size=(new_batch_size, nwp_width, nwp_height - height, n_chans), device=sat_data.device) - sat_data = torch.cat([sat_data, sat_data_zeros], dim=1) + sat_data = torch.cat([sat_data, sat_data_zeros], dim=2) assert nwp_width == width, f'widths should be the same({nwp_width},{width})' assert nwp_height == height, f'heights should be the same({nwp_height},{height})' From 2f45fb13286875d1fb97a96d188a854aee4729b2 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 12:58:25 +0000 Subject: [PATCH 05/19] tidy assert --- .../models/perceiver/perceiver_conv3d_nwp_sat.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index e8d59f3..070ca2d 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -176,9 +176,10 @@ def forward(self, x): sat_data_zeros = torch.zeros(size=(new_batch_size, nwp_width, nwp_height - height, n_chans), device=sat_data.device) sat_data = torch.cat([sat_data, sat_data_zeros], dim=2) + batch_size, sat_width, sat_height, sat_n_chans = sat_data.shape - assert nwp_width == width, f'widths should be the same({nwp_width},{width})' - assert nwp_height == height, f'heights should be the same({nwp_height},{height})' + assert nwp_width == sat_height, f'widths should be the same({nwp_width},{sat_width})' + assert nwp_height == sat_height, f'heights should be the same({nwp_height},{sat_height})' data = torch.cat((sat_data, nwp_data), dim=-1) From 8ef212cc05516b0527c264be5d1c93638da552ae Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:08:29 +0000 Subject: [PATCH 06/19] update forecast and history minutes --- configs/model/perceiver_conv3d_sat_nwp.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/model/perceiver_conv3d_sat_nwp.yaml b/configs/model/perceiver_conv3d_sat_nwp.yaml index 30f5a8c..d1a2e64 100644 --- a/configs/model/perceiver_conv3d_sat_nwp.yaml +++ b/configs/model/perceiver_conv3d_sat_nwp.yaml @@ -1,7 +1,7 @@ _target_: predict_pv_yield.models.perceiver.perceiver_conv3d_nwp_sat.Model -forecast_minutes: 30 -history_minutes: 60 +forecast_minutes: 120 +history_minutes: 30 batch_size: 32 num_latents: 24 latent_dim: 24 From 8634b2ad4296eb24720fcb32d407b70e4a5e2766 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:11:29 +0000 Subject: [PATCH 07/19] fix --- predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index 070ca2d..0d8c83e 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -176,7 +176,7 @@ def forward(self, x): sat_data_zeros = torch.zeros(size=(new_batch_size, nwp_width, nwp_height - height, n_chans), device=sat_data.device) sat_data = torch.cat([sat_data, sat_data_zeros], dim=2) - batch_size, sat_width, sat_height, sat_n_chans = sat_data.shape + new_batch_size, sat_width, sat_height, sat_n_chans = sat_data.shape assert nwp_width == sat_height, f'widths should be the same({nwp_width},{sat_width})' assert nwp_height == sat_height, f'heights should be the same({nwp_height},{sat_height})' From 3c5ae4228bf047262bf1ec3922aabfd5d5cb7d3b Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:14:18 +0000 Subject: [PATCH 08/19] add batch_size --- predict_pv_yield/models/base_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/predict_pv_yield/models/base_model.py b/predict_pv_yield/models/base_model.py index db74550..fb4a7a9 100644 --- a/predict_pv_yield/models/base_model.py +++ b/predict_pv_yield/models/base_model.py @@ -220,9 +220,9 @@ def validation_step(self, batch: BatchML, batch_idx): pass # save validation results - capacity = batch.gsp.gsp_capacity[:,-self.forecast_len_30:,0].cpu().numpy() + capacity = batch.gsp.gsp_capacity[0 : self.batch_size,-self.forecast_len_30:,0].cpu().numpy() predictions = model_output.cpu().numpy() - truths = batch.gsp.gsp_yield[:, -self.forecast_len_30:, 0].cpu().numpy() + truths = batch.gsp.gsp_yield[0 : self.batch_size, -self.forecast_len_30:, 0].cpu().numpy() predictions = predictions * capacity truths = truths * capacity From a5bd380ef07255ddb8f15177c9e7fc3bd688ea10 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:15:31 +0000 Subject: [PATCH 09/19] add batch_size --- predict_pv_yield/models/base_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/predict_pv_yield/models/base_model.py b/predict_pv_yield/models/base_model.py index fb4a7a9..281ebda 100644 --- a/predict_pv_yield/models/base_model.py +++ b/predict_pv_yield/models/base_model.py @@ -229,7 +229,7 @@ def validation_step(self, batch: BatchML, batch_idx): results = make_validation_results(truths_mw=truths, predictions_mw=predictions, capacity_mwp=capacity, - gsp_ids=batch.gsp.gsp_id[:, 0].cpu(), + gsp_ids=batch.gsp.gsp_id[0:self.batch_size, 0].cpu(), batch_idx=batch_idx, t0_datetimes_utc=pd.to_datetime(batch.metadata.t0_datetime_utc)) From e39921a13c499e704bcfa9b0c76bd8370ffae158 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:16:23 +0000 Subject: [PATCH 10/19] add batch size --- predict_pv_yield/models/base_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/predict_pv_yield/models/base_model.py b/predict_pv_yield/models/base_model.py index 281ebda..d1fcd11 100644 --- a/predict_pv_yield/models/base_model.py +++ b/predict_pv_yield/models/base_model.py @@ -231,7 +231,7 @@ def validation_step(self, batch: BatchML, batch_idx): capacity_mwp=capacity, gsp_ids=batch.gsp.gsp_id[0:self.batch_size, 0].cpu(), batch_idx=batch_idx, - t0_datetimes_utc=pd.to_datetime(batch.metadata.t0_datetime_utc)) + t0_datetimes_utc=pd.to_datetime(batch.metadata.t0_datetime_utc)[0:self.batch_size]) # append so in 'validation_epoch_end' the file is saved if batch_idx == 0: From f5fc609bd5e3c3e0e8d5b19a329ac323ce261adb Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:31:33 +0000 Subject: [PATCH 11/19] use pv history --- configs/model/perceiver_conv3d_sat_nwp.yaml | 4 ++- .../perceiver/perceiver_conv3d_nwp_sat.py | 31 ++++++++++++++----- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/configs/model/perceiver_conv3d_sat_nwp.yaml b/configs/model/perceiver_conv3d_sat_nwp.yaml index d1a2e64..1b3ceb8 100644 --- a/configs/model/perceiver_conv3d_sat_nwp.yaml +++ b/configs/model/perceiver_conv3d_sat_nwp.yaml @@ -2,10 +2,12 @@ _target_: predict_pv_yield.models.perceiver.perceiver_conv3d_nwp_sat.Model forecast_minutes: 120 history_minutes: 30 -batch_size: 32 +batch_size: 16 num_latents: 24 latent_dim: 24 embedding_dem: 0 output_variable: gsp_yield conv3d_channels: 8 use_future_satellite_images: 0 +include_pv_or_gsp_yield_history: False, +include_pv_yield_history: True, diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index 0d8c83e..5d11899 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -73,6 +73,8 @@ def __init__( output_variable: str = "pv_yield", conv3d_channels: int = 16, use_future_satellite_images: bool = True, # option not to use future sat images + include_pv_or_gsp_yield_history: bool = True, + include_pv_yield_history: int = True, ): """ Idea is to have a conv3d (+max pool) layer before both sat and nwp data go into perceiver model. @@ -86,6 +88,8 @@ def __init__( self.embedding_dem = embedding_dem self.output_variable = output_variable self.use_future_satellite_images = use_future_satellite_images + self.include_pv_yield_history = include_pv_yield_history + self.include_pv_or_gsp_yield_history = include_pv_or_gsp_yield_history super().__init__() @@ -223,14 +227,25 @@ def forward(self, x): dim=2, ) - if self.output_variable == 'pv_yield': - # take the history of the pv yield of this system, - pv_yield_history = x.pv.pv_yield[0 : self.batch_size][:, : self.history_len_5 + 1, 0].unsqueeze(-1).float() - encoder_input = torch.cat((rnn_input[:, : self.history_len_5 + 1], pv_yield_history), dim=2) - elif self.output_variable == 'gsp_yield': - # take the history of the gsp yield of this system, - gsp_history = x.gsp.gsp_yield[0: self.batch_size][:, : self.history_len_30 + 1, 0].unsqueeze(-1).float() - encoder_input = torch.cat((rnn_input[:, : self.history_len_30 + 1], gsp_history), dim=2) + if self.include_pv_or_gsp_yield_history: + if self.output_variable == 'pv_yield': + # take the history of the pv yield of this system, + pv_yield_history = x.pv.pv_yield[0 : self.batch_size][:, : self.history_len_5 + 1, 0].unsqueeze(-1).float() + encoder_input = torch.cat((rnn_input[:, : self.history_len_5 + 1], pv_yield_history), dim=2) + elif self.output_variable == 'gsp_yield': + # take the history of the gsp yield of this system, + gsp_history = x.gsp.gsp_yield[0: self.batch_size][:, : self.history_len_30 + 1, 0].unsqueeze(-1).float() + encoder_input = torch.cat((rnn_input[:, : self.history_len_30 + 1], gsp_history), dim=2) + + # add the pv yield history. This can be used if trying to predict gsp + if self.include_pv_yield_history: + pv_yield_history = ( + x.pv.pv_yield.nan_to_num(nan=0.0).float() + ) + # remove future pv + pv_yield_history[:, self.history_len_5 + 1:] = 0.0 + + encoder_input = torch.cat((rnn_input, pv_yield_history), dim=2) encoder_output, encoder_hidden = self.encoder_rnn(encoder_input) decoder_output, _ = self.decoder_rnn(rnn_input[:, -self.forecast_len :], encoder_hidden) From 0b73cb1e57c9a5b79e1783698415a23819866350 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:33:08 +0000 Subject: [PATCH 12/19] update --- configs/model/perceiver_conv3d_sat_nwp.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/model/perceiver_conv3d_sat_nwp.yaml b/configs/model/perceiver_conv3d_sat_nwp.yaml index 1b3ceb8..78e0144 100644 --- a/configs/model/perceiver_conv3d_sat_nwp.yaml +++ b/configs/model/perceiver_conv3d_sat_nwp.yaml @@ -9,5 +9,5 @@ embedding_dem: 0 output_variable: gsp_yield conv3d_channels: 8 use_future_satellite_images: 0 -include_pv_or_gsp_yield_history: False, -include_pv_yield_history: True, +include_pv_or_gsp_yield_history: False +include_pv_yield_history: True From c77a5b377774e69e1b586c193f5d4238146b1883 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:33:57 +0000 Subject: [PATCH 13/19] add batch size --- predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index 5d11899..0b2c9d3 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -240,7 +240,7 @@ def forward(self, x): # add the pv yield history. This can be used if trying to predict gsp if self.include_pv_yield_history: pv_yield_history = ( - x.pv.pv_yield.nan_to_num(nan=0.0).float() + x.pv.pv_yield[:self.batch_size].nan_to_num(nan=0.0).float() ) # remove future pv pv_yield_history[:, self.history_len_5 + 1:] = 0.0 From 3104f061f7392e61139646a094cb865f0964246a Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 13:38:39 +0000 Subject: [PATCH 14/19] fix for eoncoding RNN shape --- .../models/perceiver/perceiver_conv3d_nwp_sat.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index 0b2c9d3..5ac3e5b 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -72,8 +72,8 @@ def __init__( embedding_dem: int = 16, output_variable: str = "pv_yield", conv3d_channels: int = 16, - use_future_satellite_images: bool = True, # option not to use future sat images - include_pv_or_gsp_yield_history: bool = True, + use_future_satellite_images: bool = False, # option not to use future sat images + include_pv_or_gsp_yield_history: bool = False, include_pv_yield_history: int = True, ): """ @@ -119,10 +119,16 @@ def __init__( if self.embedding_dem: self.pv_system_id_embedding = nn.Embedding(num_embeddings=940, embedding_dim=self.embedding_dem) + rnn_input_size = FC_OUTPUT_SIZE + if self.include_pv_or_gsp_yield_history: + rnn_input_size += 1 + if self.include_pv_yield_history: + rnn_input_size += 128 + # TODO: Get rid of RNNs! self.encoder_rnn = nn.GRU( # plus 1 for history - input_size=FC_OUTPUT_SIZE + 1, + input_size=rnn_input_size, hidden_size=RNN_HIDDEN_SIZE, num_layers=2, batch_first=True, From 59e104a5f68ea476236cffb5f531bfb84658a5c3 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 2 Dec 2021 16:08:57 +0000 Subject: [PATCH 15/19] add experiments --- experiments/2021-12/2021-12-01.txt | 16 ++++++++++++++++ experiments/2021-12/2021-12-02.txt | 13 +++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 experiments/2021-12/2021-12-01.txt create mode 100644 experiments/2021-12/2021-12-02.txt diff --git a/experiments/2021-12/2021-12-01.txt b/experiments/2021-12/2021-12-01.txt new file mode 100644 index 0000000..69a1107 --- /dev/null +++ b/experiments/2021-12/2021-12-01.txt @@ -0,0 +1,16 @@ +1. Compare no future satellite data, with future satellite data + +For conv3d model + +With: https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-550/charts +Not: https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-549/charts + +Validation Error +With: 0.0665 +Not: 0.0670 + +data: +nwp +sat (no hrv, no future) +no gsp history +pv history \ No newline at end of file diff --git a/experiments/2021-12/2021-12-02.txt b/experiments/2021-12/2021-12-02.txt new file mode 100644 index 0000000..ca14b0f --- /dev/null +++ b/experiments/2021-12/2021-12-02.txt @@ -0,0 +1,13 @@ +https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-571/monitoring + +run Perceiver model + +data +satdata (no future) +nwp +no gsp +historical pv + +epoch time ~ 1.5 hours + +Validation \ No newline at end of file From d457d07bc7451219fea989f8449ac53f9676786b Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 6 Dec 2021 09:22:00 +0000 Subject: [PATCH 16/19] add test datamodel back in --- predict_pv_yield/training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/predict_pv_yield/training.py b/predict_pv_yield/training.py index 9e57503..d9807cc 100644 --- a/predict_pv_yield/training.py +++ b/predict_pv_yield/training.py @@ -85,7 +85,7 @@ def train(config: DictConfig) -> Optional[float]: # Evaluate model on test set, using the best model achieved during training if config.get("test_after_training") and not config.trainer.get("fast_dev_run"): log.info("Starting testing!") - trainer.test() + trainer.test(model=model, datamodule=datamodule) # Make sure everything closed properly log.info("Finalizing!") From c50d2f3310f8dd3658d7c5469208eb1a0d031081 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 7 Dec 2021 15:25:52 +0000 Subject: [PATCH 17/19] add test_epoch_end step --- predict_pv_yield/models/base_model.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/predict_pv_yield/models/base_model.py b/predict_pv_yield/models/base_model.py index d1fcd11..3fdf79d 100644 --- a/predict_pv_yield/models/base_model.py +++ b/predict_pv_yield/models/base_model.py @@ -153,6 +153,12 @@ def training_step(self, batch, batch_idx): return self._training_or_validation_step(batch, tag="Train") def validation_step(self, batch: BatchML, batch_idx): + self.validation_or_test_step(batch, batch_idx) + + def test_step(self, batch: BatchML, batch_idx): + self.validation_or_test_step(batch, batch_idx) + + def validation_or_test_step(self, batch: BatchML, batch_idx): if type(batch) == dict: batch = BatchML(**batch) @@ -249,8 +255,14 @@ def validation_epoch_end(self, outputs): current_epoch=self.current_epoch, logger=self.logger) - def test_step(self, batch, batch_idx): - self._training_or_validation_step(batch, tag="Test") + def test_epoch_end(self, outputs): + + logger.info("Test epoch end") + + save_validation_results_to_logger(results_dfs=self.results_dfs, + results_file_name=self.results_file_name, + current_epoch=self.current_epoch, + logger=self.logger) def configure_optimizers(self): optimizer = torch.optim.Adam(self.parameters(), lr=0.0005) From 899a32b80f52dc0cdcd46a5d33325b5c2638a095 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Dec 2021 21:33:04 +0000 Subject: [PATCH 18/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- experiments/2021-12/2021-12-01.txt | 2 +- experiments/2021-12/2021-12-02.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/2021-12/2021-12-01.txt b/experiments/2021-12/2021-12-01.txt index 69a1107..9798409 100644 --- a/experiments/2021-12/2021-12-01.txt +++ b/experiments/2021-12/2021-12-01.txt @@ -13,4 +13,4 @@ data: nwp sat (no hrv, no future) no gsp history -pv history \ No newline at end of file +pv history diff --git a/experiments/2021-12/2021-12-02.txt b/experiments/2021-12/2021-12-02.txt index ca14b0f..b39a728 100644 --- a/experiments/2021-12/2021-12-02.txt +++ b/experiments/2021-12/2021-12-02.txt @@ -10,4 +10,4 @@ historical pv epoch time ~ 1.5 hours -Validation \ No newline at end of file +Validation From 80c277bef8495bf6368ce72efda0eb35290f3f71 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 15 Dec 2021 09:22:07 +0000 Subject: [PATCH 19/19] an option to load model --- configs/experiment/perceiver_conv3d_sat_nwp.yaml | 1 + predict_pv_yield/training.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/configs/experiment/perceiver_conv3d_sat_nwp.yaml b/configs/experiment/perceiver_conv3d_sat_nwp.yaml index cea2a5d..725c8cf 100644 --- a/configs/experiment/perceiver_conv3d_sat_nwp.yaml +++ b/configs/experiment/perceiver_conv3d_sat_nwp.yaml @@ -14,6 +14,7 @@ defaults: # this allows you to overwrite only specified parameters seed: 518 +load_model: /home/ec2-user/github/predict_pv_yield/logs/runs/2021-12-13/14-52-11/checkpoints/epoch_006.ckp trainer: min_epochs: 1 diff --git a/predict_pv_yield/training.py b/predict_pv_yield/training.py index d9807cc..c69cfc7 100644 --- a/predict_pv_yield/training.py +++ b/predict_pv_yield/training.py @@ -77,7 +77,9 @@ def train(config: DictConfig) -> Optional[float]: # Train the model log.info("Starting training!") - if 'validate_only' in config: + if 'load_model' is config: + model = model.load_from_checkpoint(checkpoint_path=config['load_model']) + elif 'validate_only' in config: trainer.validate(model=model, datamodule=datamodule) else: trainer.fit(model=model, datamodule=datamodule)