Merge branch 'main' into issue/torch-channels

peterdudfield · peterdudfield · commit 249dcc52019e · 2021-11-03T13:32:49.000Z
# Conflicts:
#	nowcasting_dataset/data_sources/fake.py
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,7 +1,7 @@
 [bumpversion]
 commit = True
 tag = True
-current_version = 2.0.0
+current_version = 2.0.2
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"
diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py
@@ -1,8 +1,5 @@
 """ Various DataSources """
 from nowcasting_dataset.data_sources.data_source import DataSource  # noqa: F401
-from nowcasting_dataset.data_sources.datetime.datetime_data_source import (  # noqa: F401
-    DatetimeDataSource,
-)
 from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource
 from nowcasting_dataset.data_sources.nwp.nwp_data_source import NWPDataSource
 from nowcasting_dataset.data_sources.pv.pv_data_source import PVDataSource
diff --git a/nowcasting_dataset/data_sources/datetime/__init__.py b/nowcasting_dataset/data_sources/datetime/__init__.py
diff --git a/nowcasting_dataset/data_sources/datetime/datetime_data_source.py b/nowcasting_dataset/data_sources/datetime/datetime_data_source.py
diff --git a/nowcasting_dataset/data_sources/datetime/datetime_model.py b/nowcasting_dataset/data_sources/datetime/datetime_model.py
diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py
@@ -7,7 +7,6 @@
 import xarray as xr
 
 from nowcasting_dataset.consts import NWP_VARIABLE_NAMES, SAT_VARIABLE_NAMES
-from nowcasting_dataset.data_sources.datetime.datetime_model import Datetime
 from nowcasting_dataset.data_sources.gsp.gsp_model import GSP
 from nowcasting_dataset.data_sources.metadata.metadata_model import Metadata
 from nowcasting_dataset.data_sources.nwp.nwp_model import NWP
@@ -22,16 +21,6 @@
 )
 
 
-def datetime_fake(batch_size, seq_length_5):
-    """ Create fake data """
-    xr_arrays = [create_datetime_dataset(seq_length=seq_length_5) for _ in range(batch_size)]
-
-    # make dataset
-    xr_dataset = join_list_dataset_to_batch_dataset(xr_arrays)
-
-    return Datetime(xr_dataset)
-
-
 def gsp_fake(
     batch_size,
     seq_length_30,
diff --git a/nowcasting_dataset/dataset/batch.py b/nowcasting_dataset/dataset/batch.py
@@ -11,9 +11,7 @@
 from pydantic import BaseModel, Field
 
 from nowcasting_dataset.config.model import Configuration
-from nowcasting_dataset.data_sources.datetime.datetime_model import Datetime
 from nowcasting_dataset.data_sources.fake import (
-    datetime_fake,
     gsp_fake,
     metadata_fake,
     nwp_fake,
@@ -33,7 +31,7 @@
 
 _LOG = logging.getLogger(__name__)
 
-data_sources = [Metadata, Satellite, Topographic, PV, Sun, GSP, NWP, Datetime]
+data_sources = [Metadata, Satellite, Topographic, PV, Sun, GSP, NWP]
 
 
 class Batch(BaseModel):
@@ -62,7 +60,6 @@ class Batch(BaseModel):
     sun: Optional[Sun]
     gsp: Optional[GSP]
     nwp: Optional[NWP]
-    datetime: Optional[Datetime]
 
     @property
     def data_sources(self):
@@ -74,7 +71,6 @@ def data_sources(self):
             self.sun,
             self.gsp,
             self.nwp,
-            self.datetime,
             self.metadata,
         ]
 
@@ -119,10 +115,6 @@ def fake(configuration: Configuration):
             topographic=topographic_fake(
                 batch_size=batch_size, image_size_pixels=satellite_image_size_pixels
             ),
-            datetime=datetime_fake(
-                batch_size=batch_size,
-                seq_length_5=configuration.input_data.default_seq_length_5_minutes,
-            ),
         )
 
     def save_netcdf(self, batch_i: int, path: Path):
@@ -194,7 +186,6 @@ class Example(BaseModel):
     sun: Optional[Sun]
     gsp: Optional[GSP]
     nwp: Optional[NWP]
-    datetime: Optional[Datetime]
 
     @property
     def data_sources(self):
@@ -206,6 +197,5 @@ def data_sources(self):
             self.sun,
             self.gsp,
             self.nwp,
-            self.datetime,
             self.metadata,
         ]
diff --git a/nowcasting_dataset/time.py b/nowcasting_dataset/time.py
@@ -7,7 +7,6 @@
 import numpy as np
 import pandas as pd
 import pvlib
-import xarray as xr
 
 from nowcasting_dataset import geospatial, utils
 
@@ -211,44 +210,6 @@ def get_contiguous_time_periods(
     return pd.DataFrame(periods)
 
 
-# TODO: Delete this function and tests.
-# https://github.com/openclimatefix/nowcasting_dataset/issues/208
-def datetime_features(index: pd.DatetimeIndex) -> pd.DataFrame:
-    """
-    Make datetime features, hour_of_day and day_of_year
-
-    Args:
-        index: index of datestamps
-
-    Returns: Example data with datetime features
-
-    """
-    features = {}
-    features["hour_of_day"] = index.hour + (index.minute / 60)
-    features["day_of_year"] = index.day_of_year
-    return pd.DataFrame(features, index=index).astype(np.float32)
-
-
-# TODO: Delete this function and tests.
-# https://github.com/openclimatefix/nowcasting_dataset/issues/208
-def datetime_features_in_example(index: pd.DatetimeIndex) -> xr.Dataset:
-    """
-    Make datetime features with sin and cos
-
-    Args:
-        index: index of datestamps
-
-    Returns: Example data with datetime features
-
-    """
-    dt_features = datetime_features(index)
-    dt_features["hour_of_day"] /= 24
-    dt_features["day_of_year"] /= 365
-    dt_features = utils.sin_and_cos(dt_features)
-
-    return dt_features.to_xarray()
-
-
 def make_random_time_vectors(batch_size, seq_length_5_minutes, seq_length_30_minutes):
     """
     Make random time vectors
diff --git a/nowcasting_dataset/utils.py b/nowcasting_dataset/utils.py
@@ -61,40 +61,6 @@ def scale_to_0_to_1(a: Array) -> Array:
     return a
 
 
-# TODO: Issue #170. Is this this function still used?
-def sin_and_cos(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    For every column in df, creates cols for sin and cos of that col.
-
-    Args:
-      df: Input DataFrame.  The values must be in the range [0, 1].
-
-    Raises:
-      ValueError if any value in df is not within the range [0, 1].
-
-    Returns:
-      A new DataFrame, with twice the number of columns as the input df.
-      For each col in df, the output DataFrame will have a <col name>_sin
-      and a <col_name>_cos.
-    """
-    columns = []
-    for col_name in df.columns:
-        columns.append(f"{col_name}_sin")
-        columns.append(f"{col_name}_cos")
-    output_df = pd.DataFrame(index=df.index, columns=columns, dtype=np.float32)
-    for col_name in df.columns:
-        series = df[col_name]
-        if series.min() < 0.0 or series.max() > 1.0:
-            raise ValueError(
-                f"{col_name} has values outside the range [0, 1]!"
-                f" min={series.min()}; max={series.max()}"
-            )
-        radians = series * 2 * np.pi
-        output_df[f"{col_name}_sin"] = np.sin(radians)
-        output_df[f"{col_name}_cos"] = np.cos(radians)
-    return output_df
-
-
 def get_netcdf_filename(batch_idx: int) -> str:
     """Generate full filename, excluding path."""
     assert 0 <= batch_idx < 1e6
diff --git a/setup.py b/setup.py
@@ -10,7 +10,7 @@
 
 setup(
     name="nowcasting_dataset",
-    version="2.0.0",
+    version="2.0.2",
     license="MIT",
     description="Nowcasting Dataset",
     author="Jack Kelly, Peter Dudfield, Jacob Bieker",
diff --git a/tests/data_sources/test_datasource_output.py b/tests/data_sources/test_datasource_output.py
@@ -1,5 +1,4 @@
 from nowcasting_dataset.data_sources.fake import (
-    datetime_fake,
     gsp_fake,
     nwp_fake,
     pv_fake,
@@ -9,14 +8,6 @@
 )
 
 
-def test_datetime():
-
-    s = datetime_fake(
-        batch_size=4,
-        seq_length_5=13,
-    )
-
-
 def test_gsp():
 
     s = gsp_fake(batch_size=4, seq_length_30=13, n_gsp_per_batch=32)
@@ -39,15 +30,6 @@ def test_pv():
     s = pv_fake(batch_size=4, seq_length_5=13, n_pv_systems_per_batch=128)
 
 
-# def test_nwp_pad():
-#
-#     s = PV.fake(batch_size=4, seq_length_5=13, n_pv_systems_per_batch=37).split()[0]
-#     s.to_numpy()
-#     s.pad(n_pv_systems_per_example=128)
-#
-#     assert s.pv_yield.shape == (13, 128)
-
-
 def test_satellite():
 
     s = satellite_fake(
diff --git a/tests/data_sources/test_datetime.py b/tests/data_sources/test_datetime.py
diff --git a/tests/test_time.py b/tests/test_time.py
@@ -45,17 +45,6 @@ def test_get_contiguous_time_periods_2_with_2_chunks(min_seq_length):
     pd.testing.assert_frame_equal(periods, correct_periods)
 
 
-def test_datetime_features_in_example():
-    index = pd.date_range("2020-01-01", "2020-01-06 23:00", freq="h")
-    example = nd_time.datetime_features_in_example(index)
-    assert len(example.hour_of_day_sin) == len(index)
-    for col_name in ["hour_of_day_sin", "hour_of_day_cos"]:
-        np.testing.assert_array_almost_equal(
-            getattr(example, col_name),
-            np.tile(getattr(example, col_name)[:24], reps=6),
-        )
-
-
 def test_intersection_of_2_dataframes_of_periods():
     dt = pd.Timestamp("2020-01-01 00:00")
     a = []
diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -1,7 +1,5 @@
 # noqa: D100
-import numpy as np
 import pandas as pd
-import pytest
 
 from nowcasting_dataset import utils
 
@@ -16,20 +14,6 @@ def test_is_monotically_increasing():  # noqa: D103
     assert not utils.is_monotonically_increasing(index[::-1])
 
 
-def test_sin_and_cos():  # noqa: D103
-    df = pd.DataFrame({"a": range(30), "b": np.arange(30) - 30})
-    with pytest.raises(ValueError) as _:
-        utils.sin_and_cos(pd.DataFrame({"a": [-1, 0, 1]}))
-    with pytest.raises(ValueError) as _:
-        utils.sin_and_cos(pd.DataFrame({"a": [0, 1, 2]}))
-    with pytest.raises(ValueError) as _:
-        utils.sin_and_cos(df)
-
-    rescaled = utils.scale_to_0_to_1(df)
-    sin_and_cos = utils.sin_and_cos(rescaled)
-    np.testing.assert_array_equal(sin_and_cos.columns, ["a_sin", "a_cos", "b_sin", "b_cos"])
-
-
 def test_get_netcdf_filename():  # noqa: D103
     assert utils.get_netcdf_filename(10) == "000010.nc"
 
