Merge pull request #246 from openclimatefix/issue/239-more-gsp

Issue/239 more gsp

Author: peterdudfield

nowcasting_dataset/data_sources/gsp/gsp_data_source.py

@@ -403,13 +403,13 @@ def load_solar_gsp_data(
     gsp_power = gsp_power.sel(datetime_gmt=slice(start_dt, end_dt))
 
     # only take generation data
-    gsp_power = gsp_power.generation_mw
+    gsp_power = gsp_power.generation_normalised
 
     # make dataframe with index datetime_gmt and columns og gsp_id
     gsp_power_df = gsp_power.to_dataframe()
     gsp_power_df.reset_index(inplace=True)
     gsp_power_df = gsp_power_df.pivot(
-        index="datetime_gmt", columns="gsp_id", values="generation_mw"
+        index="datetime_gmt", columns="gsp_id", values="generation_normalised"
     )
 
     # Save memory

scripts/generate_data_for_tests/get_test_data.py

@@ -82,7 +82,7 @@
 nwp_data.to_zarr(f"{local_path}/tests/data/nwp_data/test.zarr")
 
 # ### GSP data
-filename = "gs://solar-pv-nowcasting-data/PV/GSP/v1/pv_gsp.zarr"
+filename = "gs://solar-pv-nowcasting-data/PV/GSP/v2/pv_gsp.zarr"
 
 gsp_power = xr.open_dataset(filename, engine="zarr")
 gsp_power = gsp_power.sel(datetime_gmt=slice(start_dt, end_dt))

scripts/get_raw_pv_gsp_data.py

@@ -25,9 +25,9 @@
 logging.getLogger().setLevel(logging.DEBUG)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
-start = datetime(2018, 1, 1, tzinfo=pytz.utc)
-end = datetime(2021, 1, 1, tzinfo=pytz.utc)
-gcp_path = "gs://solar-pv-nowcasting-data/PV/GSP/v1"
+start = datetime(2016, 1, 1, tzinfo=pytz.utc)
+end = datetime(2021, 10, 1, tzinfo=pytz.utc)
+gcp_path = "gs://solar-pv-nowcasting-data/PV/GSP/v2"
 
 config = {"start": start, "end": end, "gcp_path": gcp_path}
 
@@ -40,20 +40,19 @@
 
 # pivot to index as datetime_gmt, and columns as gsp_id
 data_generation = data_df.pivot(index="datetime_gmt", columns="gsp_id", values="generation_mw")
-data_generation.columns = [str(col) for col in data_generation.columns]
 data_generation_xarray = xr.DataArray(
     data_generation, name="generation_mw", dims=["datetime_gmt", "gsp_id"]
 )
 
 data_capacity = data_df.pivot(
     index="datetime_gmt", columns="gsp_id", values="installedcapacity_mwp"
 )
-data_capacity.columns = [str(col) for col in data_capacity.columns]
 data_capacity_xarray = xr.DataArray(
     data_capacity, name="installedcapacity_mwp", dims=["datetime_gmt", "gsp_id"]
 )
 
 data_xarray = xr.merge([data_generation_xarray, data_capacity_xarray])
+data_xarray = data_xarray.rename({"generation_mw": "generation_normalised"})
 
 # save config to file
 with open(os.path.join(LOCAL_TEMP_PATH, "configuration.yaml"), "w+") as f:
@@ -69,3 +68,10 @@
 
 # upload to gcp
 upload_and_delete_local_files(dst_path=gcp_path, local_path=LOCAL_TEMP_PATH)
+
+
+# # code to change 'generation_mw' to 'generation_normalised'
+# data_xarray = xr.open_dataset(gcp_path + '/pv_gsp.zarr', engine="zarr")
+# data_xarray.__setitem__('gsp_id', [int(gsp_id) for gsp_id in data_xarray.gsp_id])
+# data_xarray = data_xarray.rename({"generation_mw": "generation_normalised"})
+# data_xarray.to_zarr(gcp_path + '/pv_gsp.zarr', mode="w", encoding=encoding)

tests/data/gsp/test.zarr/.zmetadata

@@ -6,7 +6,7 @@
         },
         "datetime_gmt/.zarray": {
             "chunks": [
-                49
+                26305
             ],
             "compressor": {
                 "blocksize": 0,
@@ -29,9 +29,9 @@
                 "datetime_gmt"
             ],
             "calendar": "proleptic_gregorian",
-            "units": "minutes since 2019-01-01"
+            "units": "minutes since 2018-01-01"
         },
-        "generation_mw/.zarray": {
+        "generation_normalised/.zarray": {
             "chunks": [
                 49,
                 21
@@ -53,7 +53,7 @@
             ],
             "zarr_format": 2
         },
-        "generation_mw/.zattrs": {
+        "generation_normalised/.zattrs": {
             "_ARRAY_DIMENSIONS": [
                 "datetime_gmt",
                 "gsp_id"
@@ -70,7 +70,7 @@
                 "id": "blosc",
                 "shuffle": 1
             },
-            "dtype": "<U2",
+            "dtype": "<U21",
             "fill_value": null,
             "filters": null,
             "order": "C",

tests/data/gsp/test.zarr/datetime_gmt/.zarray

@@ -1,6 +1,6 @@
 {
     "chunks": [
-        49
+        26305
     ],
     "compressor": {
         "blocksize": 0,

tests/data/gsp/test.zarr/datetime_gmt/.zattrs

@@ -3,5 +3,5 @@
         "datetime_gmt"
     ],
     "calendar": "proleptic_gregorian",
-    "units": "minutes since 2019-01-01"
+    "units": "minutes since 2018-01-01"
 }