Merge pull request #284 from openclimatefix/jack/allow-input-data-fields-to-be-None

Allow InputData fields to be None.  Also remove some unused imports

Author: JackKelly

nowcasting_dataset/config/model.py

@@ -79,7 +79,7 @@ class PV(DataSourceMixin):
         "gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_metadata.csv",
         description="The CSV file describing each PV system.",
     )
-    n_gsp_per_example: int = Field(
+    n_pv_systems_per_example: int = Field(
         DEFAULT_N_PV_SYSTEMS_PER_EXAMPLE,
         description="The number of PV systems samples per example. "
         "If there are less in the ROI then the data is padded with zeros. ",
@@ -160,12 +160,12 @@ class InputData(BaseModel):
     Input data model.
     """
 
-    pv: PV = PV()
-    satellite: Satellite = Satellite()
-    nwp: NWP = NWP()
-    gsp: GSP = GSP()
-    topographic: Topographic = Topographic()
-    sun: Sun = Sun()
+    pv: Optional[PV] = None
+    satellite: Optional[Satellite] = None
+    nwp: Optional[NWP] = None
+    gsp: Optional[GSP] = None
+    topographic: Optional[Topographic] = None
+    sun: Optional[Sun] = None
 
     default_forecast_minutes: int = Field(
         60,
@@ -194,8 +194,14 @@ def set_forecast_and_history_minutes(cls, values):
         then set them to the default values
         """
 
-        for data_source_name in ["pv", "nwp", "satellite", "gsp", "topographic", "sun"]:
+        ALL_DATA_SOURCE_NAMES = ("pv", "satellite", "nwp", "gsp", "topographic", "sun")
+        enabled_data_sources = [
+            data_source_name
+            for data_source_name in ALL_DATA_SOURCE_NAMES
+            if values[data_source_name] is not None
+        ]
 
+        for data_source_name in enabled_data_sources:
             if values[data_source_name].forecast_minutes is None:
                 values[data_source_name].forecast_minutes = values["default_forecast_minutes"]
 
@@ -204,6 +210,21 @@ def set_forecast_and_history_minutes(cls, values):
 
         return values
 
+    @classmethod
+    def set_all_to_defaults(cls):
+        """Returns an InputData instance with all fields set to their default values.
+
+        Used for unittests.
+        """
+        return cls(
+            pv=PV(),
+            satellite=Satellite(),
+            nwp=NWP(),
+            gsp=GSP(),
+            topographic=Topographic(),
+            sun=Sun(),
+        )
+
 
 class OutputData(BaseModel):
     """Output data model"""

nowcasting_dataset/data_sources/metadata/metadata_model.py

@@ -1,14 +1,5 @@
 """ Model for output of general/metadata data, useful for a batch """
-from typing import Union
-
-from nowcasting_dataset.data_sources.datasource_output import (
-    DataSourceOutput,
-)
-
-from nowcasting_dataset.time import make_random_time_vectors
-
-
-# seems to be a pandas dataseries
+from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput
 
 
 class Metadata(DataSourceOutput):
@@ -17,4 +8,4 @@ class Metadata(DataSourceOutput):
     __slots__ = ()
     _expected_dimensions = ("t0_dt",)
 
-    # todo add validation here - https://github.com/openclimatefix/nowcasting_dataset/issues/233
+    # TODO: Add validation here - https://github.com/openclimatefix/nowcasting_dataset/issues/233

nowcasting_dataset/data_sources/pv/pv_model.py

@@ -1,41 +1,18 @@
 """ Model for output of PV data """
-import logging
-
-import numpy as np
 from xarray.ufuncs import isnan, isinf
-from pydantic import Field, validator
-
-from nowcasting_dataset.consts import (
-    Array,
-    PV_YIELD,
-    PV_DATETIME_INDEX,
-    PV_SYSTEM_Y_COORDS,
-    PV_SYSTEM_X_COORDS,
-    PV_SYSTEM_ROW_NUMBER,
-    PV_SYSTEM_ID,
-)
-from nowcasting_dataset.data_sources.datasource_output import (
-    DataSourceOutput,
-)
-from nowcasting_dataset.time import make_random_time_vectors
-
-logger = logging.getLogger(__name__)
+from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput
 
 
 class PV(DataSourceOutput):
     """ Class to store PV data as a xr.Dataset with some validation """
 
-    # Use to store xr.Dataset data
-
     __slots__ = ()
     _expected_dimensions = ("time", "id")
 
     @classmethod
     def model_validation(cls, v):
-        """ Check that all values are non NaNs """
-        assert (~isnan(v.data)).all(), f"Some pv data values are NaNs"
-        assert (~isinf(v.data)).all(), f"Some pv data values are Infinite"
-
-        assert (v.data >= 0).all(), f"Some pv data values are below 0"
-
+        """ Check that all values are not Nan, Infinite, or < 0."""
+        assert (~isnan(v.data)).all(), "Some pv data values are NaNs"
+        assert (~isinf(v.data)).all(), "Some pv data values are Infinite"
+        assert (v.data >= 0).all(), "Some pv data values are below 0"
         return v

nowcasting_dataset/data_sources/satellite/satellite_model.py

@@ -1,33 +1,19 @@
 """ Model for output of satellite data """
 from __future__ import annotations
-
-import logging
-
-import numpy as np
 from xarray.ufuncs import isnan, isinf
-from pydantic import Field
-
-from nowcasting_dataset.consts import Array
-from nowcasting_dataset.data_sources.datasource_output import (
-    DataSourceOutput,
-)
-from nowcasting_dataset.time import make_random_time_vectors
-
-logger = logging.getLogger(__name__)
+from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput
 
 
 class Satellite(DataSourceOutput):
     """ Class to store satellite data as a xr.Dataset with some validation """
 
-    # Use to store xr.Dataset data
-
     __slots__ = ()
     _expected_dimensions = ("time", "x", "y", "channels")
 
     @classmethod
     def model_validation(cls, v):
-        """ Check that all values are non negative """
-        assert (~isnan(v.data)).all(), f"Some satellite data values are NaNs"
-        assert (~isinf(v.data)).all(), f"Some satellite data values are Infinite"
-        assert (v.data != -1).all(), f"Some satellite data values are -1's"
+        """ Check that all values are not NaN, Infinite, or -1."""
+        assert (~isnan(v.data)).all(), "Some satellite data values are NaNs"
+        assert (~isinf(v.data)).all(), "Some satellite data values are Infinite"
+        assert (v.data != -1).all(), "Some satellite data values are -1's"
         return v

nowcasting_dataset/data_sources/sun/sun_model.py

@@ -1,34 +1,22 @@
 """ Model for Sun features """
-import logging
-
-import numpy as np
 from xarray.ufuncs import isnan, isinf
-from pydantic import Field, validator
-
-from nowcasting_dataset.consts import Array, SUN_AZIMUTH_ANGLE, SUN_ELEVATION_ANGLE
-from nowcasting_dataset.data_sources.datasource_output import (
-    DataSourceOutput,
-)
-from nowcasting_dataset.time import make_random_time_vectors
-
-logger = logging.getLogger(__name__)
+from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput
 
 
 class Sun(DataSourceOutput):
     """ Class to store Sun data as a xr.Dataset with some validation """
 
-    # Use to store xr.Dataset data
     __slots__ = ()
     _expected_dimensions = ("time",)
 
     @classmethod
     def model_validation(cls, v):
         """ Check that all values are non NaNs """
-        assert (~isnan(v.elevation)).all(), f"Some elevation data values are NaNs"
-        assert (~isinf(v.elevation)).all(), f"Some elevation data values are Infinite"
+        assert (~isnan(v.elevation)).all(), "Some elevation data values are NaNs"
+        assert (~isinf(v.elevation)).all(), "Some elevation data values are Infinite"
 
-        assert (~isnan(v.azimuth)).all(), f"Some azimuth data values are NaNs"
-        assert (~isinf(v.azimuth)).all(), f"Some azimuth data values are Infinite"
+        assert (~isnan(v.azimuth)).all(), "Some azimuth data values are NaNs"
+        assert (~isinf(v.azimuth)).all(), "Some azimuth data values are Infinite"
 
         assert (0 <= v.azimuth).all(), f"Some azimuth data values are lower 0, {v.azimuth.min()}"
         assert (

nowcasting_dataset/data_sources/topographic/topographic_model.py

@@ -1,27 +1,17 @@
 """ Model for Topogrpahic features """
-import logging
-
-import numpy as np
 from xarray.ufuncs import isnan, isinf
-from pydantic import Field, validator
-
-from nowcasting_dataset.consts import Array
-from nowcasting_dataset.consts import TOPOGRAPHIC_DATA
 from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput
 
-logger = logging.getLogger(__name__)
-
 
 class Topographic(DataSourceOutput):
     """ Class to store topographic data as a xr.Dataset with some validation """
 
-    # Use to store xr.Dataset data
     __slots__ = ()
     _expected_dimensions = ("x", "y")
 
     @classmethod
     def model_validation(cls, v):
         """ Check that all values are non NaNs """
-        assert (~isnan(v.data)).all(), f"Some topological data values are NaNs"
-        assert (~isinf(v.data)).all(), f"Some topological data values are Infinite"
+        assert (~isnan(v.data)).all(), "Some topological data values are NaNs"
+        assert (~isinf(v.data)).all(), "Some topological data values are Infinite"
         return v

nowcasting_dataset/dataset/batch.py

@@ -87,11 +87,11 @@ def data_sources(self):
         ]
 
     @staticmethod
-    def fake(configuration: Configuration = Configuration()):
+    def fake(configuration: Configuration):
         """ Make fake batch object """
         batch_size = configuration.process.batch_size
-        satellite_image_size_pixels = configuration.input_data.satellite.satellite_image_size_pixels
-        nwp_image_size_pixels = configuration.input_data.nwp.nwp_image_size_pixels
+        satellite_image_size_pixels = 64
+        nwp_image_size_pixels = 64
 
         return Batch(
             batch_size=batch_size,

tests/dataset/test_batch.py

@@ -1,35 +1,32 @@
+import pytest
 import tempfile
 import os
-from nowcasting_dataset.config.model import Configuration
+from nowcasting_dataset.config.model import Configuration, InputData
 from nowcasting_dataset.dataset.batch import Batch
 
 
-def test_model():
-
+@pytest.fixture
+def configuration():
     con = Configuration()
+    con.input_data = InputData.set_all_to_defaults()
     con.process.batch_size = 4
-
-    _ = Batch.fake(configuration=con)
+    return con
 
 
-def test_model_save_to_netcdf():
+def test_model(configuration):
+    _ = Batch.fake(configuration=configuration)
 
-    con = Configuration()
-    con.process.batch_size = 4
 
+def test_model_save_to_netcdf(configuration):
     with tempfile.TemporaryDirectory() as dirpath:
-        Batch.fake(configuration=con).save_netcdf(path=dirpath, batch_i=0)
+        Batch.fake(configuration=configuration).save_netcdf(path=dirpath, batch_i=0)
 
         assert os.path.exists(f"{dirpath}/satellite/0.nc")
 
 
-def test_model_load_from_netcdf():
-
-    con = Configuration()
-    con.process.batch_size = 4
-
+def test_model_load_from_netcdf(configuration):
     with tempfile.TemporaryDirectory() as dirpath:
-        Batch.fake(configuration=con).save_netcdf(path=dirpath, batch_i=0)
+        Batch.fake(configuration=configuration).save_netcdf(path=dirpath, batch_i=0)
 
         batch = Batch.load_netcdf(batch_idx=0, local_netcdf_path=dirpath)