diff --git a/pyproject.toml b/pyproject.toml index 3dde4da3..7a4fcb55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dev-dependencies = [ "pandas-stubs>=2.2.2.240909", "pre-commit>=3.8.0", "pyqt6>=6.7.1", - "pytest>=8.3.4", + "pytest>=8.4.1", "ruff==0.12.1", "types-requests>=2.32.0.20240914", ] diff --git a/src/reformatters/__main__.py b/src/reformatters/__main__.py index ab71c442..421ac919 100644 --- a/src/reformatters/__main__.py +++ b/src/reformatters/__main__.py @@ -15,6 +15,7 @@ NoaaNdviCdrAnalysisDataset, ) from reformatters.contrib.uarizona.swann.analysis import UarizonaSwannAnalysisDataset +from reformatters.dwd.icon_eu.forecast import DwdIconEuForecastDataset from reformatters.example.new_dataset import initialize_new_integration from reformatters.noaa.gfs.forecast import NoaaGfsForecastDataset from reformatters.noaa.hrrr.forecast_48_hour.dynamical_dataset import ( @@ -52,6 +53,7 @@ class UpstreamGriddedZarrsDatasetStorageConfig(StorageConfig): storage_config=UpstreamGriddedZarrsDatasetStorageConfig() ), NoaaGfsForecastDataset(storage_config=SourceCoopDatasetStorageConfig()), + DwdIconEuForecastDataset(storage_config=SourceCoopDatasetStorageConfig()), NoaaHrrrForecast48HourDataset(storage_config=SourceCoopDatasetStorageConfig()), ] diff --git a/src/reformatters/dwd/__init__.py b/src/reformatters/dwd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/reformatters/dwd/icon_eu/__init__.py b/src/reformatters/dwd/icon_eu/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/reformatters/dwd/icon_eu/forecast/__init__.py b/src/reformatters/dwd/icon_eu/forecast/__init__.py new file mode 100644 index 00000000..e91d3357 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/__init__.py @@ -0,0 +1 @@ +from .dynamical_dataset import DwdIconEuForecastDataset as DwdIconEuForecastDataset diff --git a/src/reformatters/dwd/icon_eu/forecast/dynamical_dataset.py b/src/reformatters/dwd/icon_eu/forecast/dynamical_dataset.py new file mode 100644 index 00000000..2fb26ce1 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/dynamical_dataset.py @@ -0,0 +1,55 @@ +from collections.abc import Sequence + +from reformatters.common import validation +from reformatters.common.dynamical_dataset import DynamicalDataset +from reformatters.common.kubernetes import CronJob + +from .region_job import DwdIconEuForecastRegionJob, DwdIconEuForecastSourceFileCoord +from .template_config import DwdIconEuDataVar, DwdIconEuForecastTemplateConfig + + +class DwdIconEuForecastDataset( + DynamicalDataset[DwdIconEuDataVar, DwdIconEuForecastSourceFileCoord] +): + template_config: DwdIconEuForecastTemplateConfig = DwdIconEuForecastTemplateConfig() + region_job_class: type[DwdIconEuForecastRegionJob] = DwdIconEuForecastRegionJob + + def operational_kubernetes_resources(self, image_tag: str) -> Sequence[CronJob]: + """Return the kubernetes cron job definitions to operationally update and validate this dataset.""" + # operational_update_cron_job = ReformatCronJob( + # name=f"{self.dataset_id}-operational-update", + # schedule=_OPERATIONAL_CRON_SCHEDULE, + # pod_active_deadline=timedelta(minutes=30), + # image=image_tag, + # dataset_id=self.dataset_id, + # cpu="14", + # memory="30G", + # shared_memory="12G", + # ephemeral_storage="30G", + # secret_names=self.storage_config.k8s_secret_names, + # ) + # validation_cron_job = ValidationCronJob( + # name=f"{self.dataset_id}-validation", + # schedule=_VALIDATION_CRON_SCHEDULE, + # pod_active_deadline=timedelta(minutes=10), + # image=image_tag, + # dataset_id=self.dataset_id, + # cpu="1.3", + # memory="7G", + # secret_names=self.storage_config.k8s_secret_names, + # ) + + # return [operational_update_cron_job, validation_cron_job] + raise NotImplementedError( + f"Implement `operational_kubernetes_resources` on {self.__class__.__name__}" + ) + + def validators(self) -> Sequence[validation.DataValidator]: + """Return a sequence of DataValidators to run on this dataset.""" + # return ( + # validation.check_analysis_current_data, + # validation.check_analysis_recent_nans, + # ) + raise NotImplementedError( + f"Implement `validators` on {self.__class__.__name__}" + ) diff --git a/src/reformatters/dwd/icon_eu/forecast/region_job.py b/src/reformatters/dwd/icon_eu/forecast/region_job.py new file mode 100644 index 00000000..4e5de6b1 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/region_job.py @@ -0,0 +1,289 @@ +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path + +import xarray as xr + +from reformatters.common.logging import get_logger +from reformatters.common.region_job import ( + CoordinateValueOrRange, + RegionJob, + SourceFileCoord, +) +from reformatters.common.storage import StoreFactory +from reformatters.common.types import ( + AppendDim, + ArrayFloat32, + DatetimeLike, + Dim, +) + +from .template_config import DwdIconEuDataVar + +log = get_logger(__name__) + + +class DwdIconEuForecastSourceFileCoord(SourceFileCoord): + """Coordinates of a single source file to process.""" + + def get_url(self) -> str: + raise NotImplementedError("Return the URL of the source file.") + + def out_loc( + self, + ) -> Mapping[Dim, CoordinateValueOrRange]: + """ + Returns a data array indexer which identifies the region in the output dataset + to write the data from the source file. The indexer is a dict from dimension + names to coordinate values or slices. + """ + # If the names of the coordinate attributes of your SourceFileCoord subclass are also all + # dimension names in the output dataset (e.g. init_time and lead_time), + # delete this implementation and use the default implementation of this method. + # + # Examples where you would override this method: + # - An analysis dataset created from forecast data: + # return {"time": self.init_time + self.lead_time} + return super().out_loc() + + +class DwdIconEuForecastRegionJob( + RegionJob[DwdIconEuDataVar, DwdIconEuForecastSourceFileCoord] +): + # Optionally, limit the number of variables downloaded together. + # If set to a value less than len(data_vars), downloading, reading/recompressing, + # and uploading steps will be pipelined within a region job. + # 5 is a reasonable default if it is possible to download less than all + # variables in a single file (e.g. you have a grib index). + # Leave unset if you have to download a whole file to get one variable out + # to avoid re-downloading the same file multiple times. + # + # max_vars_per_download_group: ClassVar[int | None] = None + + # Implement this method only if different variables must be retrieved from different urls + # + # # @classmethod + # def source_groups( + # cls, + # data_vars: Sequence[DwdIconEuDataVar], + # ) -> Sequence[Sequence[DwdIconEuDataVar]]: + # """ + # Return groups of variables, where all variables in a group can be retrieived from the same source file. + # """ + # grouped = defaultdict(list) + # for data_var in data_vars: + # grouped[data_var.internal_attrs.file_type].append(data_var) + # return list(grouped.values()) + + # Implement this method only if specific post processing in this dataset + # requires data from outside the region defined by self.region, + # e.g. for deaccumulation or interpolation along append_dim in an analysis dataset. + # + # def get_processing_region(self) -> slice: + # """ + # Return a slice of integer offsets into self.template_ds along self.append_dim that identifies + # the region to process. In most cases this is exactly self.region, but if additional data outside + # the region is required, for example for correct interpolation or deaccumulation, this method can + # return a modified slice (e.g. `slice(self.region.start - 1, self.region.stop + 1)`). + # """ + # return self.region + + def generate_source_file_coords( + self, + processing_region_ds: xr.Dataset, + data_var_group: Sequence[DwdIconEuDataVar], + ) -> Sequence[DwdIconEuForecastSourceFileCoord]: + """Return a sequence of coords, one for each source file required to process the data covered by processing_region_ds.""" + # return [ + # DwdIconEuForecastSourceFileCoord( + # init_time=init_time, + # lead_time=lead_time, + # ) + # for init_time, lead_time in itertools.product( + # processing_region_ds["init_time"].values, + # processing_region_ds["lead_time"].values, + # ) + # ] + raise NotImplementedError( + "Return a sequence of SourceFileCoord objects, one for each source file required to process the data covered by processing_region_ds." + ) + + def download_file(self, coord: DwdIconEuForecastSourceFileCoord) -> Path: + """Download the file for the given coordinate and return the local path.""" + # return http_download_to_disk(coord.get_url(), self.dataset_id) + raise NotImplementedError( + "Download the file for the given coordinate and return the local path." + ) + + def read_data( + self, + coord: DwdIconEuForecastSourceFileCoord, + data_var: DwdIconEuDataVar, + ) -> ArrayFloat32: + """Read and return an array of data for the given variable and source file coordinate.""" + # with rasterio.open(coord.downloaded_file_path) as reader: + # TODO: make a band index based on tag matching utility function + # matching_indexes = [ + # i + # for i in range(reader.count) + # if (tags := reader.tags(i))["GRIB_ELEMENT"] + # == data_var.internal_attrs.grib_element + # and tags["GRIB_COMMENT"] == data_var.internal_attrs.grib_comment + # ] + # assert len(matching_indexes) == 1, f"Expected exactly 1 matching band, found {matching_indexes}. {data_var.internal_attrs.grib_element=}, {data_var.internal_attrs.grib_description=}, {coord.downloaded_file_path=}" + # rasterio_band_index = 1 + matching_indexes[0] # rasterio is 1-indexed + # return reader.read(rasterio_band_index, dtype=np.float32) + raise NotImplementedError( + "Read and return data for the given variable and source file coordinate." + ) + + # Implement this to apply transformations to the array (e.g. deaccumulation) + # + # def apply_data_transformations( + # self, data_array: xr.DataArray, data_var: DwdIconEuDataVar + # ) -> None: + # """ + # Apply in-place data transformations to the output data array for a given data variable. + + # This method is called after reading all data for a variable into the shared-memory array, + # and before writing shards to the output store. The default implementation applies binary + # rounding to float32 arrays if `data_var.internal_attrs.keep_mantissa_bits` is set. + + # Subclasses may override this method to implement additional transformations such as + # deaccumulation, interpolation or other custom logic. All transformations should be + # performed in-place (don't copy `data_array`, it's large). + + # Parameters + # ---------- + # data_array : xr.DataArray + # The output data array to be transformed in-place. + # data_var : DwdIconEuDataVar + # The data variable metadata object, which may contain transformation parameters. + # """ + # super().apply_data_transformations(data_array, data_var) + + def update_template_with_results( + self, process_results: Mapping[str, Sequence[DwdIconEuForecastSourceFileCoord]] + ) -> xr.Dataset: + """ + Update template dataset based on processing results. This method is called + during operational updates. + + Subclasses should implement this method to apply dataset-specific adjustments + based on the processing results. Examples include: + - Trimming dataset along append_dim to only include successfully processed data + - Loading existing coordinate values from the primary store and updating them based on results + - Updating metadata based on what was actually processed vs what was planned + + The default implementation trims along append_dim to end at the most recent + successfully processed coordinate (timestamp). + + Parameters + ---------- + process_results : Mapping[str, Sequence[DwdIconEuForecastSourceFileCoord]] + Mapping from variable names to their source file coordinates with final processing status. + + Returns + ------- + xr.Dataset + Updated template dataset reflecting the actual processing results. + """ + # The super() implementation looks like this: + # + # max_append_dim_processed = max( + # ( + # c.out_loc()[self.append_dim] # type: ignore[type-var] + # for c in chain.from_iterable(process_results.values()) + # if c.status == SourceFileStatus.Succeeded + # ), + # default=None, + # ) + # if max_append_dim_processed is None: + # # No data was processed, trim the template to stop before this job's region + # # This is using isel's exclusive slice end behavior + # return self.template_ds.isel( + # {self.append_dim: slice(None, self.region.start)} + # ) + # else: + # return self.template_ds.sel( + # {self.append_dim: slice(None, max_append_dim_processed)} + # ) + # + # If you like the above behavior, skip implementing this method. + # If you need to customize the behavior, implement this method. + + raise NotImplementedError( + "Subclasses implement update_template_with_results() with dataset-specific logic" + ) + + @classmethod + def operational_update_jobs( + cls, + primary_store_factory: StoreFactory, + tmp_store: Path, + get_template_fn: Callable[[DatetimeLike], xr.Dataset], + append_dim: AppendDim, + all_data_vars: Sequence[DwdIconEuDataVar], + reformat_job_name: str, + ) -> tuple[ + Sequence["RegionJob[DwdIconEuDataVar, DwdIconEuForecastSourceFileCoord]"], + xr.Dataset, + ]: + """ + Return the sequence of RegionJob instances necessary to update the dataset + from its current state to include the latest available data. + + Also return the template_ds, expanded along append_dim through the end of + the data to process. The dataset returned here may extend beyond the + available data at the source, in which case `update_template_with_results` + will trim the dataset to the actual data processed. + + The exact logic is dataset-specific, but it generally follows this pattern: + 1. Figure out the range of time to process: append_dim_start (inclusive) and append_dim_end (exclusive) + a. Read existing data from the primary store to determine what's already processed + b. Optionally identify recent incomplete/non-final data for reprocessing + 2. Call get_template_fn(append_dim_end) to get the template_ds + 3. Create RegionJob instances by calling cls.get_jobs(..., filter_start=append_dim_start) + + Parameters + ---------- + primary_store_factory : StoreFactory + The factory to get the primary store to read existing data from and write updates to. + tmp_store : Path + The temporary Zarr store to write into while processing. + get_template_fn : Callable[[DatetimeLike], xr.Dataset] + Function to get the template_ds for the operational update. + append_dim : AppendDim + The dimension along which data is appended (e.g., "time"). + all_data_vars : Sequence[DwdIconEuDataVar] + Sequence of all data variable configs for this dataset. + reformat_job_name : str + The name of the reformatting job, used for progress tracking. + This is often the name of the Kubernetes job, or "local". + + Returns + ------- + Sequence[RegionJob[DwdIconEuDataVar, DwdIconEuForecastSourceFileCoord]] + RegionJob instances that need processing for operational updates. + xr.Dataset + The template_ds for the operational update. + """ + # existing_ds = xr.open_zarr(primary_store_factory.store()) + # append_dim_start = existing_ds[append_dim].max() + # append_dim_end = pd.Timestamp.now() + # template_ds = get_template_fn(append_dim_end) + + # jobs = cls.get_jobs( + # kind="operational-update", + # primary_store_factory=primary_store_factory, + # tmp_store=tmp_store, + # template_ds=template_ds, + # append_dim=append_dim, + # all_data_vars=all_data_vars, + # reformat_job_name=reformat_job_name, + # filter_start=append_dim_start, + # ) + # return jobs, template_ds + + raise NotImplementedError( + "Subclasses implement operational_update_jobs() with dataset-specific logic" + ) diff --git a/src/reformatters/dwd/icon_eu/forecast/template_config.py b/src/reformatters/dwd/icon_eu/forecast/template_config.py new file mode 100644 index 00000000..7d833f16 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/template_config.py @@ -0,0 +1,622 @@ +from collections.abc import Sequence +from typing import Any + +import numpy as np +import pandas as pd +import xarray as xr +from pydantic import computed_field + +from reformatters.common.config_models import ( + BaseInternalAttrs, + Coordinate, + CoordinateAttrs, + DatasetAttributes, + DataVar, + DataVarAttrs, + Encoding, + StatisticsApproximate, +) +from reformatters.common.template_config import ( + SPATIAL_REF_COORDS, + TemplateConfig, +) +from reformatters.common.types import AppendDim, Dim, Timedelta, Timestamp +from reformatters.common.zarr import ( + BLOSC_4BYTE_ZSTD_LEVEL3_SHUFFLE, + BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE, +) + + +class DwdIconEuInternalAttrs(BaseInternalAttrs): + """Variable specific attributes used internally to drive processing. + + Not written to the dataset. + + Attributes: + grib_element (str): The name used in ICON-EU's GRIB filename for this variable. + For example, `alb_rad` (for `surface_albedo`). + """ + + grib_element: str + + +class DwdIconEuDataVar(DataVar[DwdIconEuInternalAttrs]): + pass + + +class DwdIconEuForecastTemplateConfig(TemplateConfig[DwdIconEuDataVar]): + dims: tuple[Dim, ...] = ("init_time", "lead_time", "latitude", "longitude") + append_dim: AppendDim = "init_time" + append_dim_start: Timestamp = pd.Timestamp( + "2025-08-08T00:00" # TODO: Update this when we actual deploy operationally. + ) + append_dim_frequency: Timedelta = pd.Timedelta("6h") + + @computed_field # type: ignore[prop-decorator] + @property + def dataset_attributes(self) -> DatasetAttributes: + return DatasetAttributes( + dataset_id="dwd-icon-eu-forecast", + dataset_version="0.1.0", + name="DWD ICON-EU Forecast", + description="High-resolution weather forecasts for Europe from the ICON-EU model operated by Deutscher Wetterdienst (DWD).", + attribution="DWD ICON-EU data processed by dynamical.org from DWD.", + spatial_domain="Europe", + spatial_resolution="0.0625 degrees (~7km)", + time_domain=f"Forecasts initialized {self.append_dim_start} UTC to Present", + time_resolution=f"Forecasts initialized every {self.append_dim_frequency.total_seconds() / 3600:.0f} hours", + forecast_domain="Forecast lead time 0-120 hours (0-5 days) ahead", + forecast_resolution="Forecast step 0-78 hours: hourly, 81-120 hours: 3 hourly", + ) + + def dimension_coordinates(self) -> dict[str, Any]: + """Returns a dictionary of dimension names to coordinates for the + dataset.""" + return { + self.append_dim: self.append_dim_coordinates( + self.append_dim_start + self.append_dim_frequency + ), + "lead_time": ( # Called "step" in the ICON-EU GRIB files. + pd.timedelta_range("0h", "78h", freq="1h").union( + pd.timedelta_range("81h", "120h", freq="3h") + ) + ), + # These coordinates are for the pixel centers: + "latitude": np.linspace(29.5, 70.5, 657), + "longitude": np.linspace(-23.5, 62.5, 1377), + } + + def derive_coordinates( + self, ds: xr.Dataset + ) -> dict[str, xr.DataArray | tuple[tuple[str, ...], np.ndarray[Any, Any]]]: + """Return a dictionary of non-dimension coordinates for the dataset. + + Called whenever len(ds.append_dim) changes. + """ + return { + "valid_time": ds["init_time"] + ds["lead_time"], + "ingested_forecast_length": ( + (self.append_dim,), + np.full(ds[self.append_dim].size, np.timedelta64("NaT", "ns")), + ), + "expected_forecast_length": ( + (self.append_dim,), + np.full( + ds[self.append_dim].size, + ds["lead_time"].max(), + dtype="timedelta64[ns]", + ), + ), + "spatial_ref": SPATIAL_REF_COORDS, + } + + @computed_field # type: ignore[prop-decorator] + @property + def coords(self) -> Sequence[Coordinate]: + """Define metadata and encoding for each coordinate.""" + dim_coords = self.dimension_coordinates() + append_dim_coordinate_chunk_size = self.append_dim_coordinate_chunk_size() + + return [ + Coordinate( + name=self.append_dim, + encoding=Encoding( + dtype="int64", + fill_value=0, + compressors=[BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE], + calendar="proleptic_gregorian", + units="seconds since 1970-01-01 00:00:00", + chunks=append_dim_coordinate_chunk_size, + shards=None, + ), + attrs=CoordinateAttrs( + units="seconds since 1970-01-01 00:00:00", + statistics_approximate=StatisticsApproximate( + min=dim_coords[self.append_dim].min().isoformat(), max="Present" + ), + ), + ), + Coordinate( + name="lead_time", + encoding=Encoding( + dtype="int64", + fill_value=-1, + compressors=[BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE], + units="seconds", + chunks=len(dim_coords["lead_time"]), + shards=None, + ), + attrs=CoordinateAttrs( + units="seconds", + statistics_approximate=StatisticsApproximate( + min=str(dim_coords["lead_time"].min()), + max=str(dim_coords["lead_time"].max()), + ), + ), + ), + Coordinate( + name="latitude", + encoding=Encoding( + dtype="float64", + fill_value=np.nan, + compressors=[BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE], + chunks=len(dim_coords["latitude"]), + shards=None, + ), + attrs=CoordinateAttrs( + units="degrees_north", + statistics_approximate=StatisticsApproximate( + min=float(dim_coords["latitude"].min()), + max=float(dim_coords["latitude"].max()), + ), + ), + ), + Coordinate( + name="longitude", + encoding=Encoding( + dtype="float64", + fill_value=np.nan, + compressors=[BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE], + chunks=len(dim_coords["longitude"]), + shards=None, + ), + attrs=CoordinateAttrs( + units="degrees_east", + statistics_approximate=StatisticsApproximate( + min=float(dim_coords["longitude"].min()), + max=float(dim_coords["longitude"].max()), + ), + ), + ), + Coordinate( + name="valid_time", + encoding=Encoding( + dtype="int64", + fill_value=0, + compressors=[BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE], + calendar="proleptic_gregorian", + units="seconds since 1970-01-01 00:00:00", + chunks=( + append_dim_coordinate_chunk_size, + len(dim_coords["lead_time"]), + ), + shards=None, + ), + attrs=CoordinateAttrs( + units="seconds since 1970-01-01 00:00:00", + statistics_approximate=StatisticsApproximate( + min=self.append_dim_start.isoformat(), + max="Present + 5 days", + ), + ), + ), + Coordinate( + name="ingested_forecast_length", + encoding=Encoding( + dtype="int64", + fill_value=-1, + compressors=[BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE], + units="seconds", + chunks=append_dim_coordinate_chunk_size, + shards=None, + ), + attrs=CoordinateAttrs( + units="seconds", + statistics_approximate=StatisticsApproximate( + min=str(dim_coords["lead_time"].min()), + max=str(dim_coords["lead_time"].max()), + ), + ), + ), + Coordinate( + name="expected_forecast_length", + encoding=Encoding( + dtype="int64", + fill_value=-1, + compressors=[BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE], + units="seconds", + chunks=append_dim_coordinate_chunk_size, + shards=None, + ), + attrs=CoordinateAttrs( + units="seconds", + statistics_approximate=StatisticsApproximate( + min=str(dim_coords["lead_time"].min()), + max=str(dim_coords["lead_time"].max()), + ), + ), + ), + Coordinate( + name="spatial_ref", + encoding=Encoding( + dtype="int64", + fill_value=0, + chunks=(), # Scalar coordinate + shards=None, + ), + attrs=CoordinateAttrs( + units=None, + statistics_approximate=None, + # Derived by installing xarray, cfgrib, and rioxarray, and then running: + # from pyproj import CRS + # spherical_crs = CRS.from_wkt(WKT_STRING_EXTRACTED_FROM_ICON_EU_GRIB_BY_GDALINFO) + # ds = xr.load_dataset(ICON_EU_GRIB_FILENAME_FROM_DWD, engine='cfgrib') + # ds.rio.write_crs(spherical_crs)["spatial_ref"].attrs + crs_wkt='GEOGCS["Coordinate System imported from GRIB file",DATUM["unnamed",SPHEROID["Sphere",6371229,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST]]', + semi_major_axis=6371229.0, + semi_minor_axis=6371229.0, + inverse_flattening=0.0, + reference_ellipsoid_name="Sphere", + longitude_of_prime_meridian=0.0, + prime_meridian_name="Greenwich", + geographic_crs_name="Coordinate System imported from GRIB file", + horizontal_datum_name="unnamed", + grid_mapping_name="latitude_longitude", + spatial_ref='GEOGCS["Coordinate System imported from GRIB file",DATUM["unnamed",SPHEROID["Sphere",6371229,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST]]', + comment="A perfect sphere geographic CRS with a radius of 6,371,229m, extracted from grib.", + ), + ), + ] + + @computed_field # type: ignore[prop-decorator] + @property + def data_vars(self) -> Sequence[DwdIconEuDataVar]: + """Define metadata and encoding for each data variable.""" + # Roughly 4.5MB uncompressed, 2.5MB compressed + var_chunks: dict[Dim, int] = { + "init_time": 1, + "lead_time": 93, + "latitude": 219, # 219 = 657 / 3 + "longitude": 153, # 153 = 1377 / 9 + } + # Roughly 337MB uncompressed, 67MB compressed + var_shards: dict[Dim, int] = { + "init_time": 1, + "lead_time": 93, + "latitude": 657, + "longitude": 1377, + } + + encoding_float32_default = Encoding( + dtype="float32", + fill_value=np.nan, + chunks=tuple(var_chunks[d] for d in self.dims), + shards=tuple(var_shards[d] for d in self.dims), + compressors=[BLOSC_4BYTE_ZSTD_LEVEL3_SHUFFLE], + ) + + default_keep_mantissa_bits = 7 + + return [ + # Some of the `comment` text is taken from the DWD Database Reference PDF: + # https://www.dwd.de/DWD/forschung/nwv/fepub/icon_database_main.pdf + # + # We don't include `alb_rad` (shortwave broadband albedo for + # diffuse radiation) in the Zarr because, to quote the DWD + # Database Reference: "Values over snow-free land points are based + # on a monthly mean MODIS climatology." It's much more data-efficient + # to just download those monthly means from DWD. + DwdIconEuDataVar( + name="downward_diffuse_short_wave_radiation_flux_surface", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="aswdifd_s", + long_name="Downward diffusive short wave radiation flux at surface", + units="W m-2", + step_type="avg", + standard_name="Mean surface diffuse short-wave radiation flux", # From ECMWF. + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="aswdifd_s", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="downward_direct_short_wave_radiation_flux_surface", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="aswdir_s", + long_name="Downward direct short wave radiation flux at surface", + units="W m-2", + step_type="avg", + comment=( + "Downward solar direct radiation flux at the surface, averaged over forecast time." + " This quantity is not directly provided by the radiation scheme." + " It is aposteriori diagnosed from the definition of the surface net" + " shortwave radiation flux." + ), + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="aswdir_s", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="convective_available_potential_energy", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="cape_con", + long_name="Convective available potential energy", + units="J kg-1", + step_type="instant", + comment="Convective available potential energy", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="cape_con", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="high_cloud_cover", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="clch", + long_name="High level clouds", + units="%", + step_type="instant", + comment="Cloud Cover (0 - 400 hPa). Different agencies use different short_names for this same parameter: ECMWF: HCC; WMO GRIB table: HCDC.", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="clch", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="low_cloud_cover", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="clcl", + long_name="Low level clouds", + units="%", + step_type="instant", + comment="Cloud Cover (800 hPa - Soil). Different agencies use different short_names for this same parameter: ECMWF: LCC; WMO GRIB table: LCDC.", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="clcl", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="medium_cloud_cover", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="clcm", + long_name="Mid level clouds", + units="%", + step_type="instant", + comment="Cloud Cover (400 - 800 hPa). Different agencies use different short_names for this same parameter: ECMWF: MCC; WMO GRIB table: MCDC.", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="clcm", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="total_cloud_cover", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="clct", + long_name="Total Cloud Cover", + units="%", + step_type="instant", + comment="Total cloud cover. Different agencies use different short_names for this same parameter: ECMWF: TCC; NOAA & WMO: TCDC.", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="clct", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="snow_depth", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="sde", + long_name="Snow depth", + standard_name="lwe_thickness_of_surface_snow_amount", + units="m", + step_type="instant", + comment=( + "Snow depth in m. It is diagnosed from RHO_SNOW and W_SNOW according to" + " H_SNOW = W_SNOW / RHO_SNOW and is limited to H_SNOW <= 40 m." + ), + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="h_snow", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="pressure_reduced_to_mean_sea_level", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="prmsl", + long_name="Pressure reduced to mean sea level (MSL)", + units="Pa", + step_type="instant", + comment="Surface pressure reduced to MSL", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="pmsl", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="relative_humidity_2m", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="r2", + long_name="2 metre relative humidity", + units="%", + step_type="instant", + comment="Relative humidity at 2m above ground. Other short_names used for this parameter: rh, 2r, r.", + standard_name="relative_humidity", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="relhum_2m", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="soil_water_runoff", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="watr", + long_name="Soil water runoff", + units="kg m-2", + step_type="accum", + comment="Soil water runoff (accumulated since model start)", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="runoff_g", + keep_mantissa_bits=default_keep_mantissa_bits, + deaccumulate_to_rate=True, + ), + ), + DwdIconEuDataVar( + name="surface_water_runoff", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="watr", + long_name="Surface water Runoff", + units="kg m-2", + step_type="accum", + comment=( + "Surface water runoff from interception and snow reservoir and from" + " limited infiltration rate. Sum over forecast." + ), + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="runoff_s", + keep_mantissa_bits=default_keep_mantissa_bits, + deaccumulate_to_rate=True, + ), + ), + DwdIconEuDataVar( + name="temperature_2m", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="t2m", + long_name="2 metre temperature", + units="K", + step_type="instant", + comment=( + "Temperature at 2m above ground, averaged over all tiles of a grid point. Different agencies use different short_names for this parameter: ECMWF: 2t; NOAA & DWD: t2m." + ), + standard_name="air_temperature", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="t_2m", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="precipitation_surface", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="tp", + long_name="Total Precipitation", + units="kg m**-2", + step_type="accum", + comment=( + "Total precipitation accumulated since model start." + " TOT_PREC = RAIN_GSP + SNOW_GSP + RAIN_CON + SNOW_CON." + ), + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="tot_prec", + keep_mantissa_bits=default_keep_mantissa_bits, + deaccumulate_to_rate=True, + ), + ), + DwdIconEuDataVar( + name="wind_u_10m", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="u10", + long_name="10 metre U wind component (eastward)", + units="m/s", + step_type="instant", + standard_name="eastward_wind", + comment="Zonal wind at 10m above ground", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="u_10m", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="wind_v_10m", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="v10", + long_name="10 metre V wind component (northward)", + units="m/s", + step_type="instant", + standard_name="northward_wind", + comment="Meridional wind at 10m above ground", + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="v_10m", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="maximum_wind_10m", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="i10fg", + long_name="Time-maximum instantaneous 10 metre wind gust", + units="m/s", + step_type="max", + comment=( + "Maximum wind gust at 10 m above ground. It is diagnosed from the turbulence" + " state in the atmospheric boundary layer, including a potential" + " enhancement by the SSO parameterization over mountainous terrain." + " In the presence of deep convection, it contains an additional" + " contribution due to convective gusts." + ), + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="vmax_10m", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + DwdIconEuDataVar( + name="snow_depth_water_equivalent", + encoding=encoding_float32_default, + attrs=DataVarAttrs( + short_name="sd", + long_name="Snow depth water equivalent", + units="kg m**-2", + step_type="instant", + comment=( + "Snow depth water equivalent in kg/m2." + " Set to 0 above water surfaces and snow-free land points." + ), + ), + internal_attrs=DwdIconEuInternalAttrs( + grib_element="w_snow", + keep_mantissa_bits=default_keep_mantissa_bits, + ), + ), + ] diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/convective_available_potential_energy/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/convective_available_potential_energy/zarr.json new file mode 100644 index 00000000..d9a5bf40 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/convective_available_potential_energy/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Convective available potential energy", + "short_name": "cape_con", + "units": "J kg-1", + "comment": "Convective available potential energy", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/downward_diffuse_short_wave_radiation_flux_surface/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/downward_diffuse_short_wave_radiation_flux_surface/zarr.json new file mode 100644 index 00000000..062b24d8 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/downward_diffuse_short_wave_radiation_flux_surface/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Downward diffusive short wave radiation flux at surface", + "short_name": "aswdifd_s", + "standard_name": "Mean surface diffuse short-wave radiation flux", + "units": "W m-2", + "step_type": "avg", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/downward_direct_short_wave_radiation_flux_surface/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/downward_direct_short_wave_radiation_flux_surface/zarr.json new file mode 100644 index 00000000..04cf52f3 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/downward_direct_short_wave_radiation_flux_surface/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Downward direct short wave radiation flux at surface", + "short_name": "aswdir_s", + "units": "W m-2", + "comment": "Downward solar direct radiation flux at the surface, averaged over forecast time. This quantity is not directly provided by the radiation scheme. It is aposteriori diagnosed from the definition of the surface net shortwave radiation flux.", + "step_type": "avg", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/expected_forecast_length/c/0 b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/expected_forecast_length/c/0 new file mode 100644 index 00000000..ceddf3fa Binary files /dev/null and b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/expected_forecast_length/c/0 differ diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/expected_forecast_length/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/expected_forecast_length/zarr.json new file mode 100644 index 00000000..129dc80c --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/expected_forecast_length/zarr.json @@ -0,0 +1,52 @@ +{ + "shape": [ + 1 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 21900 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "0 days 00:00:00", + "max": "5 days 00:00:00" + }, + "units": "seconds" + }, + "dimension_names": [ + "init_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/high_cloud_cover/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/high_cloud_cover/zarr.json new file mode 100644 index 00000000..4fd8a560 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/high_cloud_cover/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "High level clouds", + "short_name": "clch", + "units": "%", + "comment": "Cloud Cover (0 - 400 hPa). Different agencies use different short_names for this same parameter: ECMWF: HCC; WMO GRIB table: HCDC.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/ingested_forecast_length/c/0 b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/ingested_forecast_length/c/0 new file mode 100644 index 00000000..45c0da26 Binary files /dev/null and b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/ingested_forecast_length/c/0 differ diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/ingested_forecast_length/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/ingested_forecast_length/zarr.json new file mode 100644 index 00000000..129dc80c --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/ingested_forecast_length/zarr.json @@ -0,0 +1,52 @@ +{ + "shape": [ + 1 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 21900 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "0 days 00:00:00", + "max": "5 days 00:00:00" + }, + "units": "seconds" + }, + "dimension_names": [ + "init_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/init_time/c/0 b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/init_time/c/0 new file mode 100644 index 00000000..24763235 Binary files /dev/null and b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/init_time/c/0 differ diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/init_time/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/init_time/zarr.json new file mode 100644 index 00000000..a5780852 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/init_time/zarr.json @@ -0,0 +1,53 @@ +{ + "shape": [ + 1 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 21900 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "2025-08-08T00:00:00", + "max": "Present" + }, + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "init_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/latitude/c/0 b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/latitude/c/0 new file mode 100644 index 00000000..76ec6ca9 Binary files /dev/null and b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/latitude/c/0 differ diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/latitude/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/latitude/zarr.json new file mode 100644 index 00000000..69cca292 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/latitude/zarr.json @@ -0,0 +1,53 @@ +{ + "shape": [ + 657 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 657 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "degrees_north", + "statistics_approximate": { + "min": 29.5, + "max": 70.5 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "latitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/lead_time/c/0 b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/lead_time/c/0 new file mode 100644 index 00000000..e275c467 Binary files /dev/null and b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/lead_time/c/0 differ diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/lead_time/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/lead_time/zarr.json new file mode 100644 index 00000000..3bc64fe5 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/lead_time/zarr.json @@ -0,0 +1,52 @@ +{ + "shape": [ + 93 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 93 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "0 days 00:00:00", + "max": "5 days 00:00:00" + }, + "units": "seconds" + }, + "dimension_names": [ + "lead_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/longitude/c/0 b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/longitude/c/0 new file mode 100644 index 00000000..b342b310 Binary files /dev/null and b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/longitude/c/0 differ diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/longitude/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/longitude/zarr.json new file mode 100644 index 00000000..e37dfced --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/longitude/zarr.json @@ -0,0 +1,53 @@ +{ + "shape": [ + 1377 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "degrees_east", + "statistics_approximate": { + "min": -23.5, + "max": 62.5 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/low_cloud_cover/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/low_cloud_cover/zarr.json new file mode 100644 index 00000000..f02c8a2e --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/low_cloud_cover/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Low level clouds", + "short_name": "clcl", + "units": "%", + "comment": "Cloud Cover (800 hPa - Soil). Different agencies use different short_names for this same parameter: ECMWF: LCC; WMO GRIB table: LCDC.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/maximum_wind_10m/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/maximum_wind_10m/zarr.json new file mode 100644 index 00000000..5533ded4 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/maximum_wind_10m/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Time-maximum instantaneous 10 metre wind gust", + "short_name": "i10fg", + "units": "m/s", + "comment": "Maximum wind gust at 10 m above ground. It is diagnosed from the turbulence state in the atmospheric boundary layer, including a potential enhancement by the SSO parameterization over mountainous terrain. In the presence of deep convection, it contains an additional contribution due to convective gusts.", + "step_type": "max", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/medium_cloud_cover/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/medium_cloud_cover/zarr.json new file mode 100644 index 00000000..52b5b6aa --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/medium_cloud_cover/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Mid level clouds", + "short_name": "clcm", + "units": "%", + "comment": "Cloud Cover (400 - 800 hPa). Different agencies use different short_names for this same parameter: ECMWF: MCC; WMO GRIB table: MCDC.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/precipitation_surface/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/precipitation_surface/zarr.json new file mode 100644 index 00000000..619936dd --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/precipitation_surface/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Total Precipitation", + "short_name": "tp", + "units": "kg m**-2", + "comment": "Total precipitation accumulated since model start. TOT_PREC = RAIN_GSP + SNOW_GSP + RAIN_CON + SNOW_CON.", + "step_type": "accum", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/pressure_reduced_to_mean_sea_level/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/pressure_reduced_to_mean_sea_level/zarr.json new file mode 100644 index 00000000..c0eab0e5 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/pressure_reduced_to_mean_sea_level/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Pressure reduced to mean sea level (MSL)", + "short_name": "prmsl", + "units": "Pa", + "comment": "Surface pressure reduced to MSL", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/relative_humidity_2m/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/relative_humidity_2m/zarr.json new file mode 100644 index 00000000..9fdc3a7c --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/relative_humidity_2m/zarr.json @@ -0,0 +1,89 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "2 metre relative humidity", + "short_name": "r2", + "standard_name": "relative_humidity", + "units": "%", + "comment": "Relative humidity at 2m above ground. Other short_names used for this parameter: rh, 2r, r.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/snow_depth/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/snow_depth/zarr.json new file mode 100644 index 00000000..b227939e --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/snow_depth/zarr.json @@ -0,0 +1,89 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Snow depth", + "short_name": "sde", + "standard_name": "lwe_thickness_of_surface_snow_amount", + "units": "m", + "comment": "Snow depth in m. It is diagnosed from RHO_SNOW and W_SNOW according to H_SNOW = W_SNOW / RHO_SNOW and is limited to H_SNOW <= 40 m.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/snow_depth_water_equivalent/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/snow_depth_water_equivalent/zarr.json new file mode 100644 index 00000000..4c6991ed --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/snow_depth_water_equivalent/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Snow depth water equivalent", + "short_name": "sd", + "units": "kg m**-2", + "comment": "Snow depth water equivalent in kg/m2. Set to 0 above water surfaces and snow-free land points.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/soil_water_runoff/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/soil_water_runoff/zarr.json new file mode 100644 index 00000000..0a070f03 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/soil_water_runoff/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Soil water runoff", + "short_name": "watr", + "units": "kg m-2", + "comment": "Soil water runoff (accumulated since model start)", + "step_type": "accum", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/spatial_ref/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/spatial_ref/zarr.json new file mode 100644 index 00000000..aa70cba4 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/spatial_ref/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "comment": "A perfect sphere geographic CRS with a radius of 6,371,229m, extracted from grib.", + "crs_wkt": "GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST]]", + "semi_major_axis": 6371229.0, + "semi_minor_axis": 6371229.0, + "inverse_flattening": 0.0, + "reference_ellipsoid_name": "Sphere", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "Coordinate System imported from GRIB file", + "horizontal_datum_name": "unnamed", + "grid_mapping_name": "latitude_longitude", + "spatial_ref": "GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST]]" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/surface_water_runoff/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/surface_water_runoff/zarr.json new file mode 100644 index 00000000..0ea8e4da --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/surface_water_runoff/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Surface water Runoff", + "short_name": "watr", + "units": "kg m-2", + "comment": "Surface water runoff from interception and snow reservoir and from limited infiltration rate. Sum over forecast.", + "step_type": "accum", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/temperature_2m/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/temperature_2m/zarr.json new file mode 100644 index 00000000..53692173 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/temperature_2m/zarr.json @@ -0,0 +1,89 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "2 metre temperature", + "short_name": "t2m", + "standard_name": "air_temperature", + "units": "K", + "comment": "Temperature at 2m above ground, averaged over all tiles of a grid point. Different agencies use different short_names for this parameter: ECMWF: 2t; NOAA & DWD: t2m.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/total_cloud_cover/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/total_cloud_cover/zarr.json new file mode 100644 index 00000000..a789754e --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/total_cloud_cover/zarr.json @@ -0,0 +1,88 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Total Cloud Cover", + "short_name": "clct", + "units": "%", + "comment": "Total cloud cover. Different agencies use different short_names for this same parameter: ECMWF: TCC; NOAA & WMO: TCDC.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/valid_time/c/0/0 b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/valid_time/c/0/0 new file mode 100644 index 00000000..5eb65ca3 Binary files /dev/null and b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/valid_time/c/0/0 differ diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/valid_time/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/valid_time/zarr.json new file mode 100644 index 00000000..77932a03 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/valid_time/zarr.json @@ -0,0 +1,56 @@ +{ + "shape": [ + 1, + 93 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 21900, + 93 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "2025-08-08T00:00:00", + "max": "Present + 5 days" + }, + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "init_time", + "lead_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/wind_u_10m/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/wind_u_10m/zarr.json new file mode 100644 index 00000000..7be962a3 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/wind_u_10m/zarr.json @@ -0,0 +1,89 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "10 metre U wind component (eastward)", + "short_name": "u10", + "standard_name": "eastward_wind", + "units": "m/s", + "comment": "Zonal wind at 10m above ground", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/wind_v_10m/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/wind_v_10m/zarr.json new file mode 100644 index 00000000..e98873b8 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/wind_v_10m/zarr.json @@ -0,0 +1,89 @@ +{ + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "10 metre V wind component (northward)", + "short_name": "v10", + "standard_name": "northward_wind", + "units": "m/s", + "comment": "Meridional wind at 10m above ground", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/zarr.json b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/zarr.json new file mode 100644 index 00000000..f39ca724 --- /dev/null +++ b/src/reformatters/dwd/icon_eu/forecast/templates/latest.zarr/zarr.json @@ -0,0 +1,2032 @@ +{ + "attributes": { + "dataset_id": "dwd-icon-eu-forecast", + "dataset_version": "0.1.0", + "name": "DWD ICON-EU Forecast", + "description": "High-resolution weather forecasts for Europe from the ICON-EU model operated by Deutscher Wetterdienst (DWD).", + "attribution": "DWD ICON-EU data processed by dynamical.org from DWD.", + "spatial_domain": "Europe", + "spatial_resolution": "0.0625 degrees (~7km)", + "time_domain": "Forecasts initialized 2025-08-08 00:00:00 UTC to Present", + "time_resolution": "Forecasts initialized every 6 hours", + "forecast_domain": "Forecast lead time 0-120 hours (0-5 days) ahead", + "forecast_resolution": "Forecast step 0-78 hours: hourly, 81-120 hours: 3 hourly" + }, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "convective_available_potential_energy": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Convective available potential energy", + "short_name": "cape_con", + "units": "J kg-1", + "comment": "Convective available potential energy", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "downward_diffuse_short_wave_radiation_flux_surface": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Downward diffusive short wave radiation flux at surface", + "short_name": "aswdifd_s", + "standard_name": "Mean surface diffuse short-wave radiation flux", + "units": "W m-2", + "step_type": "avg", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "downward_direct_short_wave_radiation_flux_surface": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Downward direct short wave radiation flux at surface", + "short_name": "aswdir_s", + "units": "W m-2", + "comment": "Downward solar direct radiation flux at the surface, averaged over forecast time. This quantity is not directly provided by the radiation scheme. It is aposteriori diagnosed from the definition of the surface net shortwave radiation flux.", + "step_type": "avg", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "expected_forecast_length": { + "shape": [ + 1 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 21900 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "0 days 00:00:00", + "max": "5 days 00:00:00" + }, + "units": "seconds" + }, + "dimension_names": [ + "init_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "high_cloud_cover": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "High level clouds", + "short_name": "clch", + "units": "%", + "comment": "Cloud Cover (0 - 400 hPa). Different agencies use different short_names for this same parameter: ECMWF: HCC; WMO GRIB table: HCDC.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "ingested_forecast_length": { + "shape": [ + 1 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 21900 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "0 days 00:00:00", + "max": "5 days 00:00:00" + }, + "units": "seconds" + }, + "dimension_names": [ + "init_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "init_time": { + "shape": [ + 1 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 21900 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "2025-08-08T00:00:00", + "max": "Present" + }, + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "init_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "latitude": { + "shape": [ + 657 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 657 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "degrees_north", + "statistics_approximate": { + "min": 29.5, + "max": 70.5 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "latitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "lead_time": { + "shape": [ + 93 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 93 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "0 days 00:00:00", + "max": "5 days 00:00:00" + }, + "units": "seconds" + }, + "dimension_names": [ + "lead_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "longitude": { + "shape": [ + 1377 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "degrees_east", + "statistics_approximate": { + "min": -23.5, + "max": 62.5 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "low_cloud_cover": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Low level clouds", + "short_name": "clcl", + "units": "%", + "comment": "Cloud Cover (800 hPa - Soil). Different agencies use different short_names for this same parameter: ECMWF: LCC; WMO GRIB table: LCDC.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "maximum_wind_10m": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Time-maximum instantaneous 10 metre wind gust", + "short_name": "i10fg", + "units": "m/s", + "comment": "Maximum wind gust at 10 m above ground. It is diagnosed from the turbulence state in the atmospheric boundary layer, including a potential enhancement by the SSO parameterization over mountainous terrain. In the presence of deep convection, it contains an additional contribution due to convective gusts.", + "step_type": "max", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "medium_cloud_cover": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Mid level clouds", + "short_name": "clcm", + "units": "%", + "comment": "Cloud Cover (400 - 800 hPa). Different agencies use different short_names for this same parameter: ECMWF: MCC; WMO GRIB table: MCDC.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "precipitation_surface": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Total Precipitation", + "short_name": "tp", + "units": "kg m**-2", + "comment": "Total precipitation accumulated since model start. TOT_PREC = RAIN_GSP + SNOW_GSP + RAIN_CON + SNOW_CON.", + "step_type": "accum", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "pressure_reduced_to_mean_sea_level": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Pressure reduced to mean sea level (MSL)", + "short_name": "prmsl", + "units": "Pa", + "comment": "Surface pressure reduced to MSL", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "relative_humidity_2m": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "2 metre relative humidity", + "short_name": "r2", + "standard_name": "relative_humidity", + "units": "%", + "comment": "Relative humidity at 2m above ground. Other short_names used for this parameter: rh, 2r, r.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "snow_depth": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Snow depth", + "short_name": "sde", + "standard_name": "lwe_thickness_of_surface_snow_amount", + "units": "m", + "comment": "Snow depth in m. It is diagnosed from RHO_SNOW and W_SNOW according to H_SNOW = W_SNOW / RHO_SNOW and is limited to H_SNOW <= 40 m.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "snow_depth_water_equivalent": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Snow depth water equivalent", + "short_name": "sd", + "units": "kg m**-2", + "comment": "Snow depth water equivalent in kg/m2. Set to 0 above water surfaces and snow-free land points.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "soil_water_runoff": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Soil water runoff", + "short_name": "watr", + "units": "kg m-2", + "comment": "Soil water runoff (accumulated since model start)", + "step_type": "accum", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "spatial_ref": { + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "comment": "A perfect sphere geographic CRS with a radius of 6,371,229m, extracted from grib.", + "crs_wkt": "GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST]]", + "semi_major_axis": 6371229.0, + "semi_minor_axis": 6371229.0, + "inverse_flattening": 0.0, + "reference_ellipsoid_name": "Sphere", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "Coordinate System imported from GRIB file", + "horizontal_datum_name": "unnamed", + "grid_mapping_name": "latitude_longitude", + "spatial_ref": "GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST]]" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "surface_water_runoff": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Surface water Runoff", + "short_name": "watr", + "units": "kg m-2", + "comment": "Surface water runoff from interception and snow reservoir and from limited infiltration rate. Sum over forecast.", + "step_type": "accum", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "temperature_2m": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "2 metre temperature", + "short_name": "t2m", + "standard_name": "air_temperature", + "units": "K", + "comment": "Temperature at 2m above ground, averaged over all tiles of a grid point. Different agencies use different short_names for this parameter: ECMWF: 2t; NOAA & DWD: t2m.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "total_cloud_cover": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Total Cloud Cover", + "short_name": "clct", + "units": "%", + "comment": "Total cloud cover. Different agencies use different short_names for this same parameter: ECMWF: TCC; NOAA & WMO: TCDC.", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "valid_time": { + "shape": [ + 1, + 93 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 21900, + 93 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "2025-08-08T00:00:00", + "max": "Present + 5 days" + }, + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "init_time", + "lead_time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "wind_u_10m": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "10 metre U wind component (eastward)", + "short_name": "u10", + "standard_name": "eastward_wind", + "units": "m/s", + "comment": "Zonal wind at 10m above ground", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "wind_v_10m": { + "shape": [ + 1, + 93, + 657, + 1377 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 93, + 657, + 1377 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 1, + 93, + 219, + 153 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "10 metre V wind component (northward)", + "short_name": "v10", + "standard_name": "northward_wind", + "units": "m/s", + "comment": "Meridional wind at 10m above ground", + "step_type": "instant", + "coordinates": "expected_forecast_length ingested_forecast_length spatial_ref valid_time", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "init_time", + "lead_time", + "latitude", + "longitude" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} \ No newline at end of file diff --git a/tests/dwd/__init__.py b/tests/dwd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/dwd/icon_eu/__init__.py b/tests/dwd/icon_eu/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/dwd/icon_eu/forecast/__init__.py b/tests/dwd/icon_eu/forecast/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/dwd/icon_eu/forecast/dynamical_dataset_test.py b/tests/dwd/icon_eu/forecast/dynamical_dataset_test.py new file mode 100644 index 00000000..8be542d5 --- /dev/null +++ b/tests/dwd/icon_eu/forecast/dynamical_dataset_test.py @@ -0,0 +1,62 @@ +# from pathlib import Path + +# import numpy as np +# import pandas as pd +# import pytest +# import xarray as xr + +# from reformatters.common import validation +# from reformatters.dwd.icon_eu.forecast.dynamical_dataset import DwdIconEuForecastDataset + +# @pytest.mark.slow +# def test_backfill_local_and_operational_update(monkeypatch: pytest.MonkeyPatch) -> None: +# dataset = DwdIconEuForecastDataset() + +# # Local backfill reformat +# dataset.backfill_local(append_dim_end=pd.Timestamp("2000-01-02")) +# ds = xr.open_zarr(dataset.primary_store_factory.store(), chunks=None) +# assert ds.time.max() == pd.Timestamp("2000-01-01") + +# # Operational update +# monkeypatch.setattr( +# dataset.region_job_class, +# "_update_append_dim_end", +# lambda: pd.Timestamp("2000-01-03"), +# ) +# monkeypatch.setattr( +# dataset.region_job_class, +# "_update_append_dim_start", +# lambda existing_ds: pd.Timestamp(existing_ds.time.max().item()), +# ) + +# dataset.update("test-update") + +# # Check resulting dataset +# updated_ds = xr.open_zarr(dataset.primary_store_factory.store(), chunks=None) + +# np.testing.assert_array_equal( +# updated_ds.time, pd.date_range("1981-10-01", "1981-10-03") +# ) +# subset_ds = updated_ds.sel(latitude=48.583335, longitude=-94, method="nearest") +# np.testing.assert_array_equal( +# subset_ds["your_variable"].values, [190.0, 163.0, 135.0] +# ) + + +# def test_operational_kubernetes_resources( +# dataset: DwdIconEuForecastDataset, +# ) -> None: +# cron_jobs = dataset.operational_kubernetes_resources("test-image-tag") + +# assert len(cron_jobs) == 2 +# update_cron_job, validation_cron_job = cron_jobs +# assert update_cron_job.name == f"{dataset.dataset_id}-operational-update" +# assert validation_cron_job.name == f"{dataset.dataset_id}-validation" +# assert update_cron_job.secret_names == dataset.storage_config.k8s_secret_names +# assert validation_cron_job.secret_names == dataset.storage_config.k8s_secret_names + + +# def test_validators(dataset: DwdIconEuForecastDataset) -> None: +# validators = tuple(dataset.validators()) +# assert len(validators) == 2 +# assert all(isinstance(v, validation.DataValidator) for v in validators) diff --git a/tests/dwd/icon_eu/forecast/region_job_test.py b/tests/dwd/icon_eu/forecast/region_job_test.py new file mode 100644 index 00000000..4c30df08 --- /dev/null +++ b/tests/dwd/icon_eu/forecast/region_job_test.py @@ -0,0 +1,37 @@ +# from unittest.mock import Mock + +# import pandas as pd + +# from reformatters.dwd.icon_eu.forecast.region_job import ( +# DwdIconEuForecastRegionJob, +# DwdIconEuForecastSourceFileCoord, +# ) +# from reformatters.dwd.icon_eu.forecast.template_config import DwdIconEuForecastTemplateConfig + +# def test_source_file_coord_get_url() -> None: +# coord = DwdIconEuForecastSourceFileCoord(time=pd.Timestamp("2000-01-01")) +# assert coord.get_url() == "https://example.com/data/2000-01-01.grib2" + + +# def test_region_job_generete_source_file_coords() -> None: +# template_config = DwdIconEuForecastTemplateConfig() +# template_ds = template_config.get_template(pd.Timestamp("2000-01-23")) + +# region_job = DwdIconEuForecastRegionJob( +# primary_store_factory=Mock(), +# tmp_store=Mock(), +# template_ds=template_ds, +# data_vars=[Mock(), Mock()], +# append_dim=template_config.append_dim, +# region=slice(0, 10), +# reformat_job_name="test", +# ) + +# processing_region_ds, output_region_ds = region_job._get_region_datasets() + +# source_file_coords = region_job.generate_source_file_coords( +# processing_region_ds, [Mock()] +# ) + +# assert len(source_file_coords) == ... +# assert ... diff --git a/tests/dwd/icon_eu/forecast/template_config_test.py b/tests/dwd/icon_eu/forecast/template_config_test.py new file mode 100644 index 00000000..e5d85314 --- /dev/null +++ b/tests/dwd/icon_eu/forecast/template_config_test.py @@ -0,0 +1,67 @@ +import json +from copy import deepcopy +from pathlib import Path + +import pandas as pd +import pytest +from pyproj import CRS + +from reformatters.dwd.icon_eu.forecast.template_config import ( + DwdIconEuForecastTemplateConfig, +) + + +def test_update_template(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Ensure that `uv run main update-template` has been run and + all changes to DwdIconEuForecastTemplateConfig are reflected in the on-disk + Zarr template.""" + template_config = DwdIconEuForecastTemplateConfig() + with open(template_config.template_path() / "zarr.json") as f: + existing_template = json.load(f) + + test_template_path = tmp_path / "latest.zarr" + monkeypatch.setattr( + DwdIconEuForecastTemplateConfig, + "template_path", + lambda _self: test_template_path, + ) + + template_config.update_template() + + with open(template_config.template_path() / "zarr.json") as f: + updated_template = json.load(f) + + assert existing_template == updated_template + + +def test_get_template_spatial_ref() -> None: + """Ensure the spatial reference system in the template matched our + expectation.""" + template_config = DwdIconEuForecastTemplateConfig() + ds = template_config.get_template( + template_config.append_dim_start + pd.Timedelta(days=10) + ) + original_attrs = deepcopy(ds.spatial_ref.attrs) + + # This WKT string is extracted from the ICON-EU GRIB by gdalinfo: + expected_crs = CRS.from_wkt("""GEOGCRS["Coordinate System imported from GRIB file", + DATUM["unnamed", + ELLIPSOID["Sphere",6371229,0, + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433, + ID["EPSG",9122]]], + CS[ellipsoidal,2], + AXIS["latitude",north, + ORDER[1], + ANGLEUNIT["degree",0.0174532925199433, + ID["EPSG",9122]]], + AXIS["longitude",east, + ORDER[2], + ANGLEUNIT["degree",0.0174532925199433, + ID["EPSG",9122]]]]""") + calculated_spatial_ref_attrs = ds.rio.write_crs(expected_crs).spatial_ref.attrs + assert set(original_attrs) - set(calculated_spatial_ref_attrs) == {"comment"} + original_attrs.pop("comment") + assert original_attrs == calculated_spatial_ref_attrs diff --git a/tests/noaa/gfs/__init__.py b/tests/noaa/gfs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uv.lock b/uv.lock index 12afa4b0..83a7f1e4 100644 --- a/uv.lock +++ b/uv.lock @@ -1174,17 +1174,18 @@ wheels = [ [[package]] name = "pytest" -version = "8.3.4" +version = "8.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, + { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/05/35/30e0d83068951d90a01852cb1cef56e5d8a09d20c7f511634cc2f7e0372a/pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761", size = 1445919, upload-time = "2024-12-01T12:54:25.98Z" } +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/92/76a1c94d3afee238333bc0a42b82935dd8f9cf8ce9e336ff87ee14d9e1cf/pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6", size = 343083, upload-time = "2024-12-01T12:54:19.735Z" }, + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, ] [[package]] @@ -1318,7 +1319,7 @@ dev = [ { name = "pandas-stubs", specifier = ">=2.2.2.240909" }, { name = "pre-commit", specifier = ">=3.8.0" }, { name = "pyqt6", specifier = ">=6.7.1" }, - { name = "pytest", specifier = ">=8.3.4" }, + { name = "pytest", specifier = ">=8.4.1" }, { name = "ruff", specifier = "==0.12.1" }, { name = "types-requests", specifier = ">=2.32.0.20240914" }, ]