Merge pull request #335 from openclimatefix/nwp-int16

nwp float16

Author: peterdudfield

nowcasting_dataset/data_sources/nwp/nwp_data_source.py

@@ -118,7 +118,7 @@ def _post_process_example(
         selected_data = selected_data.interpolate()
         selected_data = selected_data.sel(target_time=slice(start_dt, end_dt))
         selected_data = selected_data.rename({"target_time": "time", "variable": "channels"})
-        selected_data.data = selected_data.data.astype(np.float32)
+        selected_data.data = selected_data.data.astype(np.float16)
 
         return selected_data
 

nowcasting_dataset/data_sources/satellite/satellite_data_source.py

@@ -55,6 +55,7 @@ def _get_time_slice(self, t0_dt: pd.Timestamp) -> xr.DataArray:
         start_dt = self._get_start_dt(t0_dt)
         end_dt = self._get_end_dt(t0_dt)
         data = self.data.sel(time=slice(start_dt, end_dt))
+
         return data
 
     def datetime_index(self, remove_night: bool = True) -> pd.DatetimeIndex:

scripts/generate_data_for_tests/generate_satellite_test_data.py

@@ -1,21 +1,25 @@
-#!/usr/bin/env python3
+""" Make satellite test data """
+import os
 from pathlib import Path
 
 import numcodecs
 import pandas as pd
 import xarray as xr
 
+import nowcasting_dataset
 from nowcasting_dataset import consts
 
 START = pd.Timestamp("2019-01-01T12:00")
 END = pd.Timestamp("2019-01-01T18:00")
-OUTPUT_PATH = Path(__file__).parent.absolute() / ".." / "nowcasting_dataset" / "tests" / "data"
+OUTPUT_PATH = Path(os.path.dirname(nowcasting_dataset.__file__)).parent / "tests" / "data"
+print(OUTPUT_PATH)
 
 
 def generate_satellite_test_data():
+    """ Main function to make satelllite test data """
     output_filename = OUTPUT_PATH / "sat_data.zarr"
     print("Writing satellite tests data to", output_filename)
-    sat_data = xr.open_zarr(consts.SAT_DATA_ZARR, consolidated=True)
+    sat_data = xr.open_zarr(consts.SAT_FILENAME, consolidated=True)
     sat_data = sat_data.sel(variable=["HRV"], time=slice(START, END))
     encoding = {"stacked_eumetsat_data": {"compressor": numcodecs.Blosc(cname="zstd", clevel=5)}}
     sat_data = sat_data.chunk({"time": 1, "y": 704, "x": 548, "variable": 1})

scripts/generate_data_for_tests/get_test_data.py

@@ -7,6 +7,7 @@
 
 import gcsfs
 import numcodecs
+import numpy as np
 import pandas as pd
 import xarray as xr
 
@@ -70,13 +71,14 @@
     "UKV__2018-01_to_2019-12__chunks__variable10__init_time1__step1__x548__y704__.zarr"
 )
 
-nwp_data_raw = open_nwp(filename=NWP_BASE_PATH, consolidated=True)
+nwp_data_raw = open_nwp(zarr_path=NWP_BASE_PATH, consolidated=True)
 nwp_data = nwp_data_raw.sel(variable=list(NWP_VARIABLE_NAMES))
 nwp_data = nwp_data.sel(init_time=slice(start_dt, end_dt))
 nwp_data = nwp_data.sel(variable=["t"])
 nwp_data = nwp_data.sel(step=slice(nwp_data.step[0], nwp_data.step[4]))  # take 4 hours periods
 # nwp_data = nwp_data.sel(x=slice(nwp_data.x[50], nwp_data.x[100]))
 # nwp_data = nwp_data.sel(y=slice(nwp_data.y[50], nwp_data.y[100]))
+nwp_data.UKV.values = nwp_data.UKV.values.astype(np.float16)
 
 nwp_data.to_zarr(f"{local_path}/tests/data/nwp_data/test.zarr")