[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Author: pre-commit-ci[bot]

nwp/excarta/merge_excarta.py

@@ -1,21 +1,13 @@
 # import libs
-import xarray as xr
-import pandas as pd
-import numpy as np
-import datetime
 import os
-import pathlib as Path
-from datetime import datetime
-import zarr
-import ocf_blosc2
+
+import xarray as xr
 
 
 def merge_zarr_files(zarr_path, merged_zarr_path):
     # Collect paths of Zarr files in the specified directory
     zarr_files = [
-        os.path.join(zarr_path, file)
-        for file in os.listdir(zarr_path)
-        if file.endswith(".zarr")
+        os.path.join(zarr_path, file) for file in os.listdir(zarr_path) if file.endswith(".zarr")
     ]
 
     print("1")
@@ -30,7 +22,7 @@ def merge_zarr_files(zarr_path, merged_zarr_path):
 
     # Iterate over the remaining Zarr files and merge them into the initial dataset
     for file in zarr_files[1:]:
-        ds = xr.open_zarr(file)
+        xr.open_zarr(file)
         print(file)
 
         # ds_filt = ds.sel(x=slice(*x_range), y=slice(*y_range))

nwp/excarta/parse_excarta_monthly.py

@@ -1,10 +1,11 @@
 # Low memory script
+import argparse
 import os
+import pathlib
 from datetime import datetime
+
 import pandas as pd
 import xarray as xr
-import argparse
-import pathlib
 
 
 def _parse_args():
@@ -80,8 +81,7 @@ def pdtocdf(datasets):
     """
 
     datasets = [
-        ds.set_index(index=["init_time", "step", "Latitude", "Longitude"])
-        for ds in datasets
+        ds.set_index(index=["init_time", "step", "Latitude", "Longitude"]) for ds in datasets
     ]
 
     ds = xr.concat(datasets, dim="index")
@@ -109,7 +109,9 @@ def main():
         raise RuntimeError(f'Output file "{args.output}" already exist')
 
     PATH = "/mnt/storage_b/data/ocf/solar_pv_nowcasting/experimental/Excarta/sr_UK_Malta_full/solar_data"
-    month_to_process = f"{args.year}{args.month:02d}"  # combine year and month arguments into the required format
+    month_to_process = (
+        f"{args.year}{args.month:02d}"  # combine year and month arguments into the required format
+    )
     datasets = load_data_from_all_years(PATH, month_to_process)
     ds = pdtocdf(datasets)
 

nwp/excarta/parse_excarta_to_output.py

@@ -1,11 +1,11 @@
-import xarray as xr
-import pandas as pd
-import numpy as np
+import argparse
 import datetime
 import os
 import pathlib
 from datetime import datetime
-import argparse
+
+import pandas as pd
+import xarray as xr
 
 
 def _parse_args():
@@ -71,9 +71,7 @@ def pdtocdf(dfs):
     merged_df = pd.concat(dfs, ignore_index=True)
 
     ds = xr.Dataset.from_dataframe(merged_df)
-    ds = ds.set_index(index=["init_time", "step", "Latitude", "Longitude"]).unstack(
-        "index"
-    )
+    ds = ds.set_index(index=["init_time", "step", "Latitude", "Longitude"]).unstack("index")
     ds = ds.drop_vars(["LocationId", "DateTimeUTC"])
 
     var_names = ds.data_vars

nwp/excarta/parse_excarta_to_output_low_mem.py

@@ -1,10 +1,11 @@
 # Low memory script
+import argparse
 import os
+import pathlib
 from datetime import datetime
+
 import pandas as pd
 import xarray as xr
-import argparse
-import pathlib
 
 
 def _parse_args():
@@ -70,8 +71,7 @@ def pdtocdf(datasets):
     #     ds = xr.merge(datasets)
 
     datasets = [
-        ds.set_index(index=["init_time", "step", "Latitude", "Longitude"])
-        for ds in datasets
+        ds.set_index(index=["init_time", "step", "Latitude", "Longitude"]) for ds in datasets
     ]
 
     ds = xr.concat(datasets, dim="index")

nwp/icon/app.py

@@ -81,20 +81,12 @@ def process_model_files(
         var_3d_list = GLOBAL_VAR3D_LIST
         var_2d_list = GLOBAL_VAR2D_LIST
         lon_ds = xr.open_dataset(
-            list(
-                glob(
-                    os.path.join(folder, run, f"{var_base}_time-invariant_*_CLON.grib2")
-                )
-            )[0],
+            list(glob(os.path.join(folder, run, f"{var_base}_time-invariant_*_CLON.grib2")))[0],
             engine="cfgrib",
             backend_kwargs={"errors": "ignore"},
         )
         lat_ds = xr.open_dataset(
-            list(
-                glob(
-                    os.path.join(folder, run, f"{var_base}_time-invariant_*_CLAT.grib2")
-                )
-            )[0],
+            list(glob(os.path.join(folder, run, f"{var_base}_time-invariant_*_CLAT.grib2")))[0],
             engine="cfgrib",
             backend_kwargs={"errors": "ignore"},
         )
@@ -156,9 +148,7 @@ def process_model_files(
         print(var_2d)
         try:
             ds = xr.open_mfdataset(
-                os.path.join(
-                    folder, run, f"{var_base}_single-level_*_*_{var_2d.upper()}.grib2"
-                ),
+                os.path.join(folder, run, f"{var_base}_single-level_*_*_{var_2d.upper()}.grib2"),
                 engine="cfgrib",
                 combine="nested",
                 concat_dim="step",
@@ -203,9 +193,7 @@ def upload_to_hf(dataset_xr, folder, model="global", run="00", token=None):
             "longitude": 350,
             "isobaricInhPa": -1,
         }
-    encoding = {
-        var: {"compressor": Blosc2("zstd", clevel=9)} for var in dataset_xr.data_vars
-    }
+    encoding = {var: {"compressor": Blosc2("zstd", clevel=9)} for var in dataset_xr.data_vars}
     encoding["time"] = {"units": "nanoseconds since 1970-01-01"}
     with zarr.ZipStore(
         zarr_path,

scripts/convert_icon_archive.py

@@ -6,8 +6,11 @@
 
 """
 
+import multiprocessing as mp
 import os
+import subprocess
 from glob import glob
+from pathlib import Path
 
 import xarray as xr
 import zarr
@@ -18,10 +21,7 @@
     EU_VAR2D_LIST,
     EU_VAR3D_LIST,
 )
-import subprocess
 
-from pathlib import Path
-import multiprocessing as mp
 
 def decompress(full_bzip_filename: Path, temp_pth: Path) -> str:
     """
@@ -38,7 +38,7 @@ def decompress(full_bzip_filename: Path, temp_pth: Path) -> str:
     base_nat_filename = os.path.splitext(base_bzip_filename)[0]
     full_nat_filename = os.path.join(temp_pth, base_nat_filename)
     if os.path.exists(full_nat_filename):
-        return full_nat_filename # Don't decompress a second time
+        return full_nat_filename  # Don't decompress a second time
     with open(full_nat_filename, "wb") as nat_file_handler:
         process = subprocess.run(
             ["pbzip2", "--decompress", "--keep", "--stdout", full_bzip_filename],
@@ -179,8 +179,8 @@ def upload_to_hf(dataset_xr, folder, model="eu", run="00", token=None):
     encoding = {var: {"compressor": Blosc2("zstd", clevel=9)} for var in dataset_xr.data_vars}
     encoding["time"] = {"units": "nanoseconds since 1970-01-01"}
     with zarr.ZipStore(
-            zarr_path,
-            mode="w",
+        zarr_path,
+        mode="w",
     ) as store:
         dataset_xr.chunk(chunking).to_zarr(store, encoding=encoding, compute=True)
     done = False
@@ -189,10 +189,10 @@ def upload_to_hf(dataset_xr, folder, model="eu", run="00", token=None):
             api.upload_file(
                 path_or_fileobj=zarr_path,
                 path_in_repo=f"data/{dataset_xr.time.dt.year.values}/"
-                             f"{dataset_xr.time.dt.month.values}/"
-                             f"{dataset_xr.time.dt.day.values}/"
-                             f"{dataset_xr.time.dt.year.values}{str(dataset_xr.time.dt.month.values).zfill(2)}{str(dataset_xr.time.dt.day.values).zfill(2)}"
-                             f"_{str(dataset_xr.time.dt.hour.values).zfill(2)}.zarr.zip",
+                f"{dataset_xr.time.dt.month.values}/"
+                f"{dataset_xr.time.dt.day.values}/"
+                f"{dataset_xr.time.dt.year.values}{str(dataset_xr.time.dt.month.values).zfill(2)}{str(dataset_xr.time.dt.day.values).zfill(2)}"
+                f"_{str(dataset_xr.time.dt.hour.values).zfill(2)}.zarr.zip",
                 repo_id="openclimatefix/dwd-icon-global"
                 if model == "global"
                 else "openclimatefix/dwd-icon-eu",