Move BatchML/Datasets/Datamodule to Nowcasting-dataloader (#243)

* Move models to nowcasting-dataloader

#major

* Move tests

* Fix import error

* Move SatelliteML test

* Fix import error

* Remove batch test data

* Move test data generation script

* Move test generation script back

* Update get_test_data.py

Author: jacobbieker

.github/workflows/python-app.yml

@@ -3,7 +3,7 @@
 
 name: Python Tests
 
-on: [push, pull_request]
+on: [push]
 
 jobs:
   build:

nowcasting_dataset/data_sources/datasource_output.py

@@ -53,69 +53,7 @@ def save_netcdf(self, batch_i: int, path: Path):
         self.to_netcdf(local_filename, engine="h5netcdf", mode="w", encoding=encoding)
 
 
-class DataSourceOutputML(BaseModel):
-    """General Data Source output pydantic class.
-
-    Data source output classes should inherit from this class
-    """
-
-    class Config:
-        """Allowed classes e.g. tensor.Tensor"""
-
-        # TODO maybe there is a better way to do this
-        arbitrary_types_allowed = True
-
-    batch_size: int = Field(
-        0,
-        ge=0,
-        description="The size of this batch. If the batch size is 0, "
-        "then this item stores one data item i.e Example",
-    )
-
-    def get_name(self) -> str:
-        """Get the name of the class"""
-        return self.__class__.__name__.lower()
-
-    def get_datetime_index(self):
-        """Datetime index for the data"""
-        pass
-
-
 def pad_nans(array, pad_width) -> np.ndarray:
     """Pad nans with nans"""
     array = array.astype(np.float32)
     return np.pad(array, pad_width, constant_values=np.NaN)
-
-
-def pad_data(
-    data: DataSourceOutputML,
-    pad_size: int,
-    one_dimensional_arrays: List[str],
-    two_dimensional_arrays: List[str],
-):
-    """
-    Pad (if necessary) so returned arrays are always of size
-
-    data has two types of arrays in it, one dimensional arrays and two dimensional arrays
-    the one dimensional arrays are padded in that dimension
-    the two dimensional arrays are padded in the second dimension
-
-    Note that class is edited so nothing is returned.
-
-    Args:
-        data: typed dictionary of data objects
-        pad_size: the maount that should be padded
-        one_dimensional_arrays: list of data items that should be padded by one dimension
-        two_dimensional_arrays: list of data tiems that should be padded in the third dimension (and more)
-
-    """
-    # Pad (if necessary) so returned arrays are always of size
-    pad_shape = (0, pad_size)  # (before, after)
-
-    for name in one_dimensional_arrays:
-        data.__setattr__(name, pad_nans(data.__getattribute__(name), pad_width=pad_shape))
-
-    for variable in two_dimensional_arrays:
-        data.__setattr__(
-            variable, pad_nans(data.__getattribute__(variable), pad_width=((0, 0), pad_shape))
-        )  # (axis0, axis1)

nowcasting_dataset/data_sources/datetime/datetime_model.py

@@ -1,11 +1,5 @@
 """ Model for output of datetime data """
-import numpy as np
-import xarray as xr
-from pydantic import validator
-
-from nowcasting_dataset.consts import Array, DATETIME_FEATURE_NAMES
 from nowcasting_dataset.data_sources.datasource_output import (
-    DataSourceOutputML,
     DataSourceOutput,
 )
 
@@ -21,96 +15,3 @@ class Datetime(DataSourceOutput):
 
     # todo add validation here - https://github.com/openclimatefix/nowcasting_dataset/issues/233
     _expected_dimensions = ("time",)
-
-
-class DatetimeML(DataSourceOutputML):
-    """ Model for output of datetime data """
-
-    hour_of_day_sin: Array  #: Shape: [batch_size,] seq_length
-    hour_of_day_cos: Array
-    day_of_year_sin: Array
-    day_of_year_cos: Array
-    datetime_index: Array
-
-    @property
-    def sequence_length(self):
-        """The sequence length of the pv data"""
-        return self.hour_of_day_sin.shape[-1]
-
-    @validator("hour_of_day_cos")
-    def v_hour_of_day_cos(cls, v, values):
-        """ Validate 'hour_of_day_cos' """
-        assert v.shape[-1] == values["hour_of_day_sin"].shape[-1]
-        return v
-
-    @validator("day_of_year_sin")
-    def v_day_of_year_sin(cls, v, values):
-        """ Validate 'day_of_year_sin' """
-        assert v.shape[-1] == values["hour_of_day_sin"].shape[-1]
-        return v
-
-    @validator("day_of_year_cos")
-    def v_day_of_year_cos(cls, v, values):
-        """ Validate 'day_of_year_cos' """
-        assert v.shape[-1] == values["hour_of_day_sin"].shape[-1]
-        return v
-
-    @staticmethod
-    def fake(batch_size, seq_length_5):
-        """ Make a fake Datetime object """
-        return DatetimeML(
-            batch_size=batch_size,
-            hour_of_day_sin=np.random.randn(
-                batch_size,
-                seq_length_5,
-            ),
-            hour_of_day_cos=np.random.randn(
-                batch_size,
-                seq_length_5,
-            ),
-            day_of_year_sin=np.random.randn(
-                batch_size,
-                seq_length_5,
-            ),
-            day_of_year_cos=np.random.randn(
-                batch_size,
-                seq_length_5,
-            ),
-            datetime_index=np.sort(np.random.randn(batch_size, seq_length_5))[:, ::-1].copy(),
-            # copy is needed as torch doesnt not support negative strides
-        )
-
-    def to_xr_dataset(self, _):
-        """ Make a xr dataset """
-        individual_datasets = []
-        for name in DATETIME_FEATURE_NAMES:
-
-            var = self.__getattribute__(name)
-
-            data = xr.DataArray(
-                var,
-                dims=["time"],
-                coords={"time": self.datetime_index},
-                name=name,
-            )
-
-            ds = data.to_dataset()
-            ds = coord_to_range(ds, "time", prefix=None)
-            individual_datasets.append(ds)
-
-        return xr.merge(individual_datasets)
-
-    @staticmethod
-    def from_xr_dataset(xr_dataset):
-        """ Change xr dataset to model. If data does not exist, then return None """
-        if "hour_of_day_sin" in xr_dataset.keys():
-            return DatetimeML(
-                batch_size=xr_dataset["hour_of_day_sin"].shape[0],
-                hour_of_day_sin=xr_dataset["hour_of_day_sin"],
-                hour_of_day_cos=xr_dataset["hour_of_day_cos"],
-                day_of_year_sin=xr_dataset["day_of_year_sin"],
-                day_of_year_cos=xr_dataset["day_of_year_cos"],
-                datetime_index=xr_dataset["hour_of_day_sin"].time,
-            )
-        else:
-            return None

nowcasting_dataset/data_sources/gsp/gsp_model.py

@@ -1,19 +1,7 @@
 """ Model for output of GSP data """
 import logging
 
-import numpy as np
-from pydantic import Field, validator
-
-from nowcasting_dataset.consts import Array
-from nowcasting_dataset.consts import (
-    GSP_ID,
-    GSP_YIELD,
-    GSP_X_COORDS,
-    GSP_Y_COORDS,
-    GSP_DATETIME_INDEX,
-)
 from nowcasting_dataset.data_sources.datasource_output import (
-    DataSourceOutputML,
     DataSourceOutput,
 )
 from nowcasting_dataset.time import make_random_time_vectors
@@ -28,109 +16,3 @@ class GSP(DataSourceOutput):
     _expected_dimensions = ("time", "id")
 
     # todo add validation here - https://github.com/openclimatefix/nowcasting_dataset/issues/233
-
-
-class GSPML(DataSourceOutputML):
-    """ Model for output of GSP data """
-
-    # Shape: [batch_size,] seq_length, width, height, channel
-    gsp_yield: Array = Field(
-        ...,
-        description=" GSP yield from all GSP in the region of interest (ROI). \
-    : Includes central GSP system, which will always be the first entry. \
-    : shape = [batch_size, ] seq_length, n_gsp_per_example",
-    )
-
-    #: GSP identification.
-    #: shape = [batch_size, ] n_pv_systems_per_example
-    gsp_id: Array = Field(..., description="gsp id from NG")
-
-    gsp_datetime_index: Array = Field(
-        ...,
-        description="The datetime associated with the gsp data. shape = [batch_size, ] sequence length,",
-    )
-
-    gsp_x_coords: Array = Field(
-        ...,
-        description="The x (OSGB geo-spatial) coordinates of the gsp. "
-        "This is in fact the x centroid of the GSP region"
-        "Shape: [batch_size,] n_gsp_per_example",
-    )
-    gsp_y_coords: Array = Field(
-        ...,
-        description="The y (OSGB geo-spatial) coordinates of the gsp. "
-        "This are in fact the y centroid of the GSP region"
-        "Shape: [batch_size,] n_gsp_per_example",
-    )
-
-    @property
-    def number_of_gsp(self):
-        """The number of Grid Supply Points in this example"""
-        return self.gsp_yield.shape[-1]
-
-    @property
-    def sequence_length(self):
-        """The sequence length of the GSP PV power timeseries data"""
-        return self.gsp_yield.shape[-2]
-
-    @validator("gsp_yield")
-    def gsp_yield_shape(cls, v, values):
-        """ Validate 'gsp_yield' """
-        if values["batch_size"] > 0:
-            assert len(v.shape) == 3
-        else:
-            assert len(v.shape) == 2
-        return v
-
-    @validator("gsp_x_coords")
-    def x_coordinates_shape(cls, v, values):
-        """ Validate 'gsp_x_coords' """
-        assert v.shape[-1] == values["gsp_yield"].shape[-1]
-        return v
-
-    @validator("gsp_y_coords")
-    def y_coordinates_shape(cls, v, values):
-        """ Validate 'gsp_y_coords' """
-        assert v.shape[-1] == values["gsp_yield"].shape[-1]
-        return v
-
-    @staticmethod
-    def fake(batch_size, seq_length_30, n_gsp_per_batch, time_30=None):
-        """ Make a fake GSP object """
-        if time_30 is None:
-            _, _, time_30 = make_random_time_vectors(
-                batch_size=batch_size, seq_length_5_minutes=0, seq_length_30_minutes=seq_length_30
-            )
-
-        return GSPML(
-            batch_size=batch_size,
-            gsp_yield=np.random.randn(
-                batch_size,
-                seq_length_30,
-                n_gsp_per_batch,
-            ),
-            gsp_id=np.sort(np.random.randint(0, 340, (batch_size, n_gsp_per_batch))),
-            gsp_datetime_index=time_30,
-            gsp_x_coords=np.sort(np.random.randn(batch_size, n_gsp_per_batch)),
-            gsp_y_coords=np.sort(np.random.randn(batch_size, n_gsp_per_batch))[:, ::-1].copy(),
-        )
-        # copy is needed as torch doesnt not support negative strides
-
-    def get_datetime_index(self) -> Array:
-        """ Get the datetime index of this data """
-        return self.gsp_datetime_index
-
-    @staticmethod
-    def from_xr_dataset(xr_dataset):
-        """ Change xr dataset to model. If data does not exist, then return None """
-        if "gsp_yield" in xr_dataset.keys():
-            return GSPML(
-                batch_size=xr_dataset["gsp_yield"].shape[0],
-                gsp_yield=xr_dataset[GSP_YIELD],
-                gsp_id=xr_dataset[GSP_ID],
-                gsp_datetime_index=xr_dataset[GSP_DATETIME_INDEX],
-                gsp_x_coords=xr_dataset[GSP_X_COORDS],
-                gsp_y_coords=xr_dataset[GSP_Y_COORDS],
-            )
-        else:
-            return None

nowcasting_dataset/data_sources/metadata/metadata_model.py

@@ -1,13 +1,7 @@
 """ Model for output of general/metadata data, useful for a batch """
 from typing import Union
 
-import numpy as np
-import torch
-import xarray as xr
-from pydantic import Field
-
 from nowcasting_dataset.data_sources.datasource_output import (
-    DataSourceOutputML,
     DataSourceOutput,
 )
 
@@ -24,50 +18,3 @@ class Metadata(DataSourceOutput):
     _expected_dimensions = ("t0_dt",)
 
     # todo add validation here - https://github.com/openclimatefix/nowcasting_dataset/issues/233
-
-
-class MetadataML(DataSourceOutputML):
-    """Model for output of general/metadata data"""
-
-    # TODO add descriptions
-    t0_dt: Union[xr.DataArray, np.ndarray, torch.Tensor, int]  #: Shape: [batch_size,]
-    x_meters_center: Union[xr.DataArray, np.ndarray, torch.Tensor, int]
-    y_meters_center: Union[xr.DataArray, np.ndarray, torch.Tensor, int]
-    object_at_center_label: Union[xr.DataArray, np.ndarray, torch.Tensor, int] = Field(
-        ...,
-        description="What object is at the center of the batch data "
-        "0: Nothing at the center, "
-        "1: GSP system, "
-        "2: PV system",
-    )
-
-    @staticmethod
-    def fake(batch_size, t0_dt=None):
-        """Make a xr dataset"""
-        if t0_dt is None:
-            t0_dt, _, _ = make_random_time_vectors(
-                batch_size=batch_size, seq_length_5_minutes=0, seq_length_30_minutes=0
-            )
-
-        return MetadataML(
-            batch_size=batch_size,
-            t0_dt=t0_dt,
-            x_meters_center=np.random.randn(
-                batch_size,
-            ),
-            y_meters_center=np.random.randn(
-                batch_size,
-            ),
-            object_at_center_label=np.array([1] * batch_size),
-        )
-
-    @staticmethod
-    def from_xr_dataset(xr_dataset):
-        """Change xr dataset to model. If data does not exist, then return None"""
-        return MetadataML(
-            batch_size=xr_dataset.t0_dt.shape[0],
-            t0_dt=xr_dataset.t0_dt.values,
-            x_meters_center=xr_dataset.x_meters_center.values,
-            y_meters_center=xr_dataset.y_meters_center.values,
-            object_at_center_label=xr_dataset.object_at_center_label.values,
-        )