diff --git a/configs/experiment/perceiver_conv3d_sat_nwp.yaml b/configs/experiment/perceiver_conv3d_sat_nwp.yaml index cea2a5d..725c8cf 100644 --- a/configs/experiment/perceiver_conv3d_sat_nwp.yaml +++ b/configs/experiment/perceiver_conv3d_sat_nwp.yaml @@ -14,6 +14,7 @@ defaults: # this allows you to overwrite only specified parameters seed: 518 +load_model: /home/ec2-user/github/predict_pv_yield/logs/runs/2021-12-13/14-52-11/checkpoints/epoch_006.ckp trainer: min_epochs: 1 diff --git a/configs/model/perceiver_conv3d_sat_nwp.yaml b/configs/model/perceiver_conv3d_sat_nwp.yaml index 30f5a8c..78e0144 100644 --- a/configs/model/perceiver_conv3d_sat_nwp.yaml +++ b/configs/model/perceiver_conv3d_sat_nwp.yaml @@ -1,11 +1,13 @@ _target_: predict_pv_yield.models.perceiver.perceiver_conv3d_nwp_sat.Model -forecast_minutes: 30 -history_minutes: 60 -batch_size: 32 +forecast_minutes: 120 +history_minutes: 30 +batch_size: 16 num_latents: 24 latent_dim: 24 embedding_dem: 0 output_variable: gsp_yield conv3d_channels: 8 use_future_satellite_images: 0 +include_pv_or_gsp_yield_history: False +include_pv_yield_history: True diff --git a/experiments/2021-12/2021-12-01.txt b/experiments/2021-12/2021-12-01.txt new file mode 100644 index 0000000..9798409 --- /dev/null +++ b/experiments/2021-12/2021-12-01.txt @@ -0,0 +1,16 @@ +1. Compare no future satellite data, with future satellite data + +For conv3d model + +With: https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-550/charts +Not: https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-549/charts + +Validation Error +With: 0.0665 +Not: 0.0670 + +data: +nwp +sat (no hrv, no future) +no gsp history +pv history diff --git a/experiments/2021-12/2021-12-02.txt b/experiments/2021-12/2021-12-02.txt new file mode 100644 index 0000000..b39a728 --- /dev/null +++ b/experiments/2021-12/2021-12-02.txt @@ -0,0 +1,13 @@ +https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-571/monitoring + +run Perceiver model + +data +satdata (no future) +nwp +no gsp +historical pv + +epoch time ~ 1.5 hours + +Validation diff --git a/predict_pv_yield/models/base_model.py b/predict_pv_yield/models/base_model.py index db74550..3fdf79d 100644 --- a/predict_pv_yield/models/base_model.py +++ b/predict_pv_yield/models/base_model.py @@ -153,6 +153,12 @@ def training_step(self, batch, batch_idx): return self._training_or_validation_step(batch, tag="Train") def validation_step(self, batch: BatchML, batch_idx): + self.validation_or_test_step(batch, batch_idx) + + def test_step(self, batch: BatchML, batch_idx): + self.validation_or_test_step(batch, batch_idx) + + def validation_or_test_step(self, batch: BatchML, batch_idx): if type(batch) == dict: batch = BatchML(**batch) @@ -220,18 +226,18 @@ def validation_step(self, batch: BatchML, batch_idx): pass # save validation results - capacity = batch.gsp.gsp_capacity[:,-self.forecast_len_30:,0].cpu().numpy() + capacity = batch.gsp.gsp_capacity[0 : self.batch_size,-self.forecast_len_30:,0].cpu().numpy() predictions = model_output.cpu().numpy() - truths = batch.gsp.gsp_yield[:, -self.forecast_len_30:, 0].cpu().numpy() + truths = batch.gsp.gsp_yield[0 : self.batch_size, -self.forecast_len_30:, 0].cpu().numpy() predictions = predictions * capacity truths = truths * capacity results = make_validation_results(truths_mw=truths, predictions_mw=predictions, capacity_mwp=capacity, - gsp_ids=batch.gsp.gsp_id[:, 0].cpu(), + gsp_ids=batch.gsp.gsp_id[0:self.batch_size, 0].cpu(), batch_idx=batch_idx, - t0_datetimes_utc=pd.to_datetime(batch.metadata.t0_datetime_utc)) + t0_datetimes_utc=pd.to_datetime(batch.metadata.t0_datetime_utc)[0:self.batch_size]) # append so in 'validation_epoch_end' the file is saved if batch_idx == 0: @@ -249,8 +255,14 @@ def validation_epoch_end(self, outputs): current_epoch=self.current_epoch, logger=self.logger) - def test_step(self, batch, batch_idx): - self._training_or_validation_step(batch, tag="Test") + def test_epoch_end(self, outputs): + + logger.info("Test epoch end") + + save_validation_results_to_logger(results_dfs=self.results_dfs, + results_file_name=self.results_file_name, + current_epoch=self.current_epoch, + logger=self.logger) def configure_optimizers(self): optimizer = torch.optim.Adam(self.parameters(), lr=0.0005) diff --git a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py index 8f77fbb..5ac3e5b 100644 --- a/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py +++ b/predict_pv_yield/models/perceiver/perceiver_conv3d_nwp_sat.py @@ -72,7 +72,9 @@ def __init__( embedding_dem: int = 16, output_variable: str = "pv_yield", conv3d_channels: int = 16, - use_future_satellite_images: bool = True, # option not to use future sat images + use_future_satellite_images: bool = False, # option not to use future sat images + include_pv_or_gsp_yield_history: bool = False, + include_pv_yield_history: int = True, ): """ Idea is to have a conv3d (+max pool) layer before both sat and nwp data go into perceiver model. @@ -86,6 +88,8 @@ def __init__( self.embedding_dem = embedding_dem self.output_variable = output_variable self.use_future_satellite_images = use_future_satellite_images + self.include_pv_yield_history = include_pv_yield_history + self.include_pv_or_gsp_yield_history = include_pv_or_gsp_yield_history super().__init__() @@ -115,10 +119,16 @@ def __init__( if self.embedding_dem: self.pv_system_id_embedding = nn.Embedding(num_embeddings=940, embedding_dim=self.embedding_dem) + rnn_input_size = FC_OUTPUT_SIZE + if self.include_pv_or_gsp_yield_history: + rnn_input_size += 1 + if self.include_pv_yield_history: + rnn_input_size += 128 + # TODO: Get rid of RNNs! self.encoder_rnn = nn.GRU( # plus 1 for history - input_size=FC_OUTPUT_SIZE + 1, + input_size=rnn_input_size, hidden_size=RNN_HIDDEN_SIZE, num_layers=2, batch_first=True, @@ -169,8 +179,17 @@ def forward(self, x): nwp_data = nwp_data.reshape(new_batch_size, nwp_width, nwp_height, n_nwp_chans) - assert nwp_width == width, f'widths should be the same({nwp_width},{width})' - assert nwp_height == height, f'heights should be the same({nwp_height},{height})' + # v15 the width and height are a lot less, so lets expand the sat data. There should be a better way + sat_data_zeros = torch.zeros(size=(new_batch_size, nwp_width - width, height, n_chans), + device=sat_data.device) + sat_data = torch.cat([sat_data, sat_data_zeros], dim=1) + sat_data_zeros = torch.zeros(size=(new_batch_size, nwp_width, nwp_height - height, n_chans), + device=sat_data.device) + sat_data = torch.cat([sat_data, sat_data_zeros], dim=2) + new_batch_size, sat_width, sat_height, sat_n_chans = sat_data.shape + + assert nwp_width == sat_height, f'widths should be the same({nwp_width},{sat_width})' + assert nwp_height == sat_height, f'heights should be the same({nwp_height},{sat_height})' data = torch.cat((sat_data, nwp_data), dim=-1) @@ -214,14 +233,25 @@ def forward(self, x): dim=2, ) - if self.output_variable == 'pv_yield': - # take the history of the pv yield of this system, - pv_yield_history = x.pv.pv_yield[0 : self.batch_size][:, : self.history_len_5 + 1, 0].unsqueeze(-1).float() - encoder_input = torch.cat((rnn_input[:, : self.history_len_5 + 1], pv_yield_history), dim=2) - elif self.output_variable == 'gsp_yield': - # take the history of the gsp yield of this system, - gsp_history = x.gsp.gsp_yield[0: self.batch_size][:, : self.history_len_30 + 1, 0].unsqueeze(-1).float() - encoder_input = torch.cat((rnn_input[:, : self.history_len_30 + 1], gsp_history), dim=2) + if self.include_pv_or_gsp_yield_history: + if self.output_variable == 'pv_yield': + # take the history of the pv yield of this system, + pv_yield_history = x.pv.pv_yield[0 : self.batch_size][:, : self.history_len_5 + 1, 0].unsqueeze(-1).float() + encoder_input = torch.cat((rnn_input[:, : self.history_len_5 + 1], pv_yield_history), dim=2) + elif self.output_variable == 'gsp_yield': + # take the history of the gsp yield of this system, + gsp_history = x.gsp.gsp_yield[0: self.batch_size][:, : self.history_len_30 + 1, 0].unsqueeze(-1).float() + encoder_input = torch.cat((rnn_input[:, : self.history_len_30 + 1], gsp_history), dim=2) + + # add the pv yield history. This can be used if trying to predict gsp + if self.include_pv_yield_history: + pv_yield_history = ( + x.pv.pv_yield[:self.batch_size].nan_to_num(nan=0.0).float() + ) + # remove future pv + pv_yield_history[:, self.history_len_5 + 1:] = 0.0 + + encoder_input = torch.cat((rnn_input, pv_yield_history), dim=2) encoder_output, encoder_hidden = self.encoder_rnn(encoder_input) decoder_output, _ = self.decoder_rnn(rnn_input[:, -self.forecast_len :], encoder_hidden) diff --git a/predict_pv_yield/training.py b/predict_pv_yield/training.py index 9e57503..c69cfc7 100644 --- a/predict_pv_yield/training.py +++ b/predict_pv_yield/training.py @@ -77,7 +77,9 @@ def train(config: DictConfig) -> Optional[float]: # Train the model log.info("Starting training!") - if 'validate_only' in config: + if 'load_model' is config: + model = model.load_from_checkpoint(checkpoint_path=config['load_model']) + elif 'validate_only' in config: trainer.validate(model=model, datamodule=datamodule) else: trainer.fit(model=model, datamodule=datamodule) @@ -85,7 +87,7 @@ def train(config: DictConfig) -> Optional[float]: # Evaluate model on test set, using the best model achieved during training if config.get("test_after_training") and not config.trainer.get("fast_dev_run"): log.info("Starting testing!") - trainer.test() + trainer.test(model=model, datamodule=datamodule) # Make sure everything closed properly log.info("Finalizing!")